/**
 * Model Performance Leaderboard
 *
 * Tracks model performance metrics over time for routing decisions.
 *
 * @module multimodel/leaderboard
 */
import { readFile, writeFile, mkdir, access } from "fs/promises";
import { join, dirname } from "path";
import { randomUUID } from "crypto";
const DEFAULT_LEADERBOARD = {
    updatedAt: new Date().toISOString(),
    totalRuns: 0,
    runs: [],
    modelStats: {},
};
const MAX_STORED_RUNS = 1000;
const LEADERBOARD_FILE = ".vaspera/model-scores.json";
/**
 * Load leaderboard from file
 */
async function loadLeaderboard(projectPath) {
    const filePath = join(projectPath, LEADERBOARD_FILE);
    try {
        await access(filePath);
        const content = await readFile(filePath, "utf-8");
        return JSON.parse(content);
    }
    catch {
        return { ...DEFAULT_LEADERBOARD };
    }
}
/**
 * Save leaderboard to file
 */
async function saveLeaderboard(projectPath, data) {
    const filePath = join(projectPath, LEADERBOARD_FILE);
    await mkdir(dirname(filePath), { recursive: true });
    await writeFile(filePath, JSON.stringify(data, null, 2));
}
/**
 * Record a model run
 */
export async function recordModelRun(projectPath, run) {
    const leaderboard = await loadLeaderboard(projectPath);
    const fullRun = {
        ...run,
        runId: randomUUID().slice(0, 8),
        timestamp: new Date().toISOString(),
    };
    // Add run to recent runs (keep only MAX_STORED_RUNS)
    leaderboard.runs.unshift(fullRun);
    if (leaderboard.runs.length > MAX_STORED_RUNS) {
        leaderboard.runs = leaderboard.runs.slice(0, MAX_STORED_RUNS);
    }
    leaderboard.totalRuns++;
    leaderboard.updatedAt = fullRun.timestamp;
    // Recalculate stats for this model
    const modelRuns = leaderboard.runs.filter((r) => r.modelId === run.modelId);
    leaderboard.modelStats[run.modelId] = calculateModelStats(run.modelId, modelRuns);
    await saveLeaderboard(projectPath, leaderboard);
    return fullRun;
}
/**
 * Calculate aggregated stats for a model
 */
function calculateModelStats(modelId, runs) {
    if (runs.length === 0) {
        return {
            modelId,
            totalRuns: 0,
            precision: 0,
            recallEstimate: 0,
            f1Score: 0,
            avgResponseTimeMs: 0,
            avgCostPerRun: 0,
            totalBySeverity: { critical: 0, high: 0, medium: 0, low: 0, info: 0 },
            lastRunAt: new Date().toISOString(),
            trend: "stable",
        };
    }
    const severities = ["critical", "high", "medium", "low", "info"];
    // Calculate precision (confirmed / reported)
    const totalReported = runs.reduce((sum, r) => sum + r.findingsReported, 0);
    const totalConfirmed = runs.reduce((sum, r) => sum + r.findingsConfirmed, 0);
    const precision = totalReported > 0 ? totalConfirmed / totalReported : 0;
    // Estimate recall (we don't have ground truth, so use confirmed as proxy)
    // A model with high confirmation rate that finds many issues has good recall
    const avgConfirmed = totalConfirmed / runs.length;
    const maxConfirmedAnyRun = Math.max(...runs.map((r) => r.findingsConfirmed), 1);
    const recallEstimate = avgConfirmed / maxConfirmedAnyRun;
    // F1 score
    const f1Score = precision + recallEstimate > 0
        ? 2 * (precision * recallEstimate) / (precision + recallEstimate)
        : 0;
    // Averages
    const avgResponseTimeMs = runs.reduce((sum, r) => sum + r.responseTimeMs, 0) / runs.length;
    const avgCostPerRun = runs.reduce((sum, r) => sum + r.cost, 0) / runs.length;
    // By severity totals
    const totalBySeverity = {
        critical: 0,
        high: 0,
        medium: 0,
        low: 0,
        info: 0,
    };
    for (const run of runs) {
        for (const sev of severities) {
            totalBySeverity[sev] += run.bySeverity[sev] || 0;
        }
    }
    // Calculate trend (compare recent half to older half)
    let trend = "stable";
    if (runs.length >= 4) {
        const midpoint = Math.floor(runs.length / 2);
        const recentRuns = runs.slice(0, midpoint);
        const olderRuns = runs.slice(midpoint);
        const recentPrecision = calculatePrecision(recentRuns);
        const olderPrecision = calculatePrecision(olderRuns);
        if (recentPrecision > olderPrecision * 1.05) {
            trend = "improving";
        }
        else if (recentPrecision < olderPrecision * 0.95) {
            trend = "declining";
        }
    }
    return {
        modelId,
        totalRuns: runs.length,
        precision: Math.round(precision * 100) / 100,
        recallEstimate: Math.round(recallEstimate * 100) / 100,
        f1Score: Math.round(f1Score * 100) / 100,
        avgResponseTimeMs: Math.round(avgResponseTimeMs),
        avgCostPerRun: Math.round(avgCostPerRun * 100) / 100,
        totalBySeverity,
        lastRunAt: runs[0]?.timestamp || new Date().toISOString(),
        trend,
    };
}
function calculatePrecision(runs) {
    const totalReported = runs.reduce((sum, r) => sum + r.findingsReported, 0);
    const totalConfirmed = runs.reduce((sum, r) => sum + r.findingsConfirmed, 0);
    return totalReported > 0 ? totalConfirmed / totalReported : 0;
}
/**
 * Get the leaderboard
 */
export async function getLeaderboard(projectPath) {
    return loadLeaderboard(projectPath);
}
/**
 * Get ranked models by a specific metric
 */
export async function getRankedModels(projectPath, options = {}) {
    const { metric = "f1Score", limit = 10, minRuns = 1 } = options;
    const leaderboard = await loadLeaderboard(projectPath);
    const models = Object.values(leaderboard.modelStats)
        .filter((m) => m.totalRuns >= minRuns)
        .sort((a, b) => {
        // Higher is better for f1, precision, recall
        // Lower is better for cost and response time
        if (metric === "avgCostPerRun" || metric === "avgResponseTimeMs") {
            return a[metric] - b[metric];
        }
        return b[metric] - a[metric];
    })
        .slice(0, limit);
    return models.map((m, i) => ({ ...m, rank: i + 1 }));
}
/**
 * Get model recommendation for an agent based on performance
 */
export async function getRecommendedModel(projectPath, agent, options = {}) {
    const { prioritize = "accuracy", minRuns = 2 } = options;
    const leaderboard = await loadLeaderboard(projectPath);
    // Filter runs by agent and get model stats
    const agentRuns = leaderboard.runs.filter((r) => r.agent === agent);
    const modelIds = [...new Set(agentRuns.map((r) => r.modelId))];
    const agentModelStats = modelIds
        .map((modelId) => {
        const runs = agentRuns.filter((r) => r.modelId === modelId);
        return calculateModelStats(modelId, runs);
    })
        .filter((s) => s.totalRuns >= minRuns);
    if (agentModelStats.length === 0) {
        return {
            recommended: null,
            reason: `No models have at least ${minRuns} runs for agent: ${agent}`,
            alternatives: [],
        };
    }
    // Sort by priority
    let sorted;
    switch (prioritize) {
        case "speed":
            sorted = agentModelStats.sort((a, b) => a.avgResponseTimeMs - b.avgResponseTimeMs);
            break;
        case "cost":
            sorted = agentModelStats.sort((a, b) => a.avgCostPerRun - b.avgCostPerRun);
            break;
        default:
            sorted = agentModelStats.sort((a, b) => b.f1Score - a.f1Score);
    }
    const recommended = sorted[0];
    const alternatives = sorted.slice(1, 4).map((m) => m.modelId);
    let reason;
    switch (prioritize) {
        case "speed":
            reason = `Fastest response time (${recommended.avgResponseTimeMs}ms avg)`;
            break;
        case "cost":
            reason = `Lowest cost ($${recommended.avgCostPerRun.toFixed(4)} avg)`;
            break;
        default:
            reason = `Highest F1 score (${(recommended.f1Score * 100).toFixed(1)}%)`;
    }
    if (recommended.trend === "improving") {
        reason += " - performance improving";
    }
    else if (recommended.trend === "declining") {
        reason += " - warning: performance declining";
    }
    return {
        recommended: recommended.modelId,
        reason,
        alternatives,
    };
}
/**
 * Format leaderboard as markdown
 */
export function formatLeaderboardAsMarkdown(data) {
    const lines = [
        "# Model Performance Leaderboard",
        "",
        `**Last Updated**: ${data.updatedAt}`,
        `**Total Runs Tracked**: ${data.totalRuns}`,
        "",
        "## Rankings by F1 Score",
        "",
        "| Rank | Model | F1 | Precision | Recall Est. | Runs | Avg Cost | Trend |",
        "|------|-------|-----|-----------|-------------|------|----------|-------|",
    ];
    const ranked = Object.values(data.modelStats)
        .sort((a, b) => b.f1Score - a.f1Score);
    ranked.forEach((model, i) => {
        const trendIcon = model.trend === "improving" ? "📈" :
            model.trend === "declining" ? "📉" : "➡️";
        lines.push(`| ${i + 1} | ${model.modelId} | ${(model.f1Score * 100).toFixed(1)}% | ${(model.precision * 100).toFixed(1)}% | ${(model.recallEstimate * 100).toFixed(1)}% | ${model.totalRuns} | $${model.avgCostPerRun.toFixed(4)} | ${trendIcon} |`);
    });
    if (ranked.length === 0) {
        lines.push("| - | No data yet | - | - | - | - | - | - |");
    }
    return lines.join("\n");
}
/**
 * Clear leaderboard data
 */
export async function clearLeaderboard(projectPath) {
    await saveLeaderboard(projectPath, { ...DEFAULT_LEADERBOARD });
}
//# sourceMappingURL=leaderboard.js.map