/**
 * Multi-Model Runner
 *
 * Orchestrates running certification agents across multiple LLM providers.
 *
 * @module multimodel/runner
 */
import { MODEL_PRICING, calculateCost, formatCost, formatTokens } from "../cost/index.js";
import { calculateMultiModelConsensus, generateConsensusSummary, } from "./consensus.js";
/**
 * Default model configurations
 */
export const DEFAULT_MODELS = [
    {
        id: "claude-3.5-sonnet",
        provider: "anthropic",
        weight: 1.0,
        enabled: true,
    },
    {
        id: "gpt-4-turbo",
        provider: "openai",
        weight: 1.0,
        enabled: true,
    },
    {
        id: "gemini-1.5-pro",
        provider: "google",
        weight: 1.0,
        enabled: true,
    },
];
/**
 * Default run options
 */
const DEFAULT_OPTIONS = {
    models: [],
    agents: ["security"],
    parallel: true,
    timeout: 300000, // 5 minutes
    failFast: false,
    minModels: 2,
    matchThreshold: 70,
    trackCosts: true,
};
/**
 * Multi-model runner for certifications
 */
export class MultiModelRunner {
    config;
    results;
    constructor(config) {
        this.config = {
            models: config?.models || DEFAULT_MODELS,
            defaults: { ...DEFAULT_OPTIONS, ...config?.defaults },
            thresholds: {
                unanimous: 100,
                majority: 50,
                goodAgreement: 0.6,
                ...config?.thresholds,
            },
        };
        this.results = new Map();
    }
    /**
     * Get enabled models
     */
    getEnabledModels() {
        return this.config.models.filter((m) => m.enabled !== false);
    }
    /**
     * Get model configuration
     */
    getModelConfig(modelId) {
        return this.config.models.find((m) => m.id === modelId);
    }
    /**
     * Enable/disable a model
     */
    setModelEnabled(modelId, enabled) {
        const model = this.config.models.find((m) => m.id === modelId);
        if (model) {
            model.enabled = enabled;
        }
    }
    /**
     * Add a model configuration
     */
    addModel(config) {
        const existing = this.config.models.findIndex((m) => m.id === config.id);
        if (existing >= 0) {
            this.config.models[existing] = config;
        }
        else {
            this.config.models.push(config);
        }
    }
    /**
     * Record a model run result
     * This is called by certification agents to record their results
     */
    recordResult(certificationId, agent, model, findings, metrics) {
        const modelConfig = this.getModelConfig(model);
        const provider = modelConfig?.provider || this.getProviderFromModel(model);
        // Calculate cost
        const cost = calculateCost(model, {
            inputTokens: metrics.inputTokens,
            outputTokens: metrics.outputTokens,
            totalTokens: metrics.inputTokens + metrics.outputTokens,
        });
        const result = {
            model,
            provider,
            agent,
            findings,
            duration: metrics.duration,
            tokens: {
                input: metrics.inputTokens,
                output: metrics.outputTokens,
                total: metrics.inputTokens + metrics.outputTokens,
            },
            cost: cost.totalCost,
            success: true,
            timestamp: new Date().toISOString(),
        };
        // Store result
        const key = `${certificationId}:${agent}`;
        const existing = this.results.get(key) || [];
        existing.push(result);
        this.results.set(key, existing);
        return result;
    }
    /**
     * Record a failed model run
     */
    recordFailure(certificationId, agent, model, error, duration) {
        const modelConfig = this.getModelConfig(model);
        const provider = modelConfig?.provider || this.getProviderFromModel(model);
        const result = {
            model,
            provider,
            agent,
            findings: [],
            duration,
            tokens: { input: 0, output: 0, total: 0 },
            cost: 0,
            success: false,
            error,
            timestamp: new Date().toISOString(),
        };
        // Store result
        const key = `${certificationId}:${agent}`;
        const existing = this.results.get(key) || [];
        existing.push(result);
        this.results.set(key, existing);
        return result;
    }
    /**
     * Get results for a certification/agent
     */
    getResults(certificationId, agent) {
        const key = `${certificationId}:${agent}`;
        return this.results.get(key) || [];
    }
    /**
     * Check if we have enough results for consensus
     */
    hasEnoughResults(certificationId, agent, minModels) {
        const results = this.getResults(certificationId, agent);
        const successfulResults = results.filter((r) => r.success);
        const required = minModels ?? this.config.defaults?.minModels ?? 2;
        return successfulResults.length >= required;
    }
    /**
     * Calculate consensus for a certification/agent
     */
    calculateConsensus(certificationId, agent, options) {
        const results = this.getResults(certificationId, agent);
        if (results.length === 0) {
            return null;
        }
        return calculateMultiModelConsensus(certificationId, agent, results, {
            matchThreshold: options?.matchThreshold ?? this.config.defaults?.matchThreshold,
            requireMajority: options?.requireMajority,
        });
    }
    /**
     * Generate summary for a certification/agent
     */
    generateSummary(certificationId, agent) {
        const results = this.getResults(certificationId, agent);
        if (results.length === 0)
            return null;
        const consensus = this.calculateConsensus(certificationId, agent);
        if (!consensus)
            return null;
        const successfulResults = results.filter((r) => r.success);
        const allFindings = successfulResults.flatMap((r) => r.findings);
        const totalCost = successfulResults.reduce((sum, r) => sum + r.cost, 0);
        const totalTokens = successfulResults.reduce((sum, r) => sum + r.tokens.total, 0);
        return {
            totalModels: results.length,
            successfulModels: successfulResults.length,
            totalFindings: allFindings.length,
            uniqueFindings: consensus.mergedFindings.length,
            agreementRate: consensus.metrics.agreementRate,
            reliability: consensus.metrics.fleissKappa,
            totalCost,
            totalTokens,
            highConfidenceFindings: consensus.metrics.unanimousFindings + consensus.metrics.majorityFindings,
            disputedFindings: consensus.metrics.disputedFindings,
        };
    }
    /**
     * Generate markdown report
     */
    generateReport(certificationId, agent) {
        const consensus = this.calculateConsensus(certificationId, agent);
        if (!consensus) {
            return `# Multi-Model Report\n\nNo results available for ${certificationId}/${agent}`;
        }
        const summary = this.generateSummary(certificationId, agent);
        const consensusReport = generateConsensusSummary(consensus);
        const lines = [
            consensusReport,
            "",
            "## Cost Summary",
            "",
        ];
        if (summary) {
            lines.push(`- **Total Cost**: ${formatCost(summary.totalCost)}`);
            lines.push(`- **Total Tokens**: ${formatTokens(summary.totalTokens)}`);
            lines.push("");
        }
        // Individual model results
        lines.push("## Individual Model Results", "");
        lines.push("| Model | Provider | Findings | Tokens | Cost | Duration |");
        lines.push("|-------|----------|----------|--------|------|----------|");
        for (const result of consensus.results) {
            const status = result.success ? "OK" : "FAIL";
            lines.push(`| ${result.model} (${status}) | ${result.provider} | ${result.findings.length} | ${formatTokens(result.tokens.total)} | ${formatCost(result.cost)} | ${result.duration}ms |`);
        }
        return lines.join("\n");
    }
    /**
     * Clear results for a certification
     */
    clearResults(certificationId, agent) {
        if (agent) {
            this.results.delete(`${certificationId}:${agent}`);
        }
        else {
            // Clear all results for this certification
            for (const key of this.results.keys()) {
                if (key.startsWith(`${certificationId}:`)) {
                    this.results.delete(key);
                }
            }
        }
    }
    /**
     * Get provider from model ID
     */
    getProviderFromModel(model) {
        const pricing = MODEL_PRICING[model];
        if (pricing) {
            return pricing.provider;
        }
        if (model.startsWith("claude"))
            return "anthropic";
        if (model.startsWith("gpt"))
            return "openai";
        if (model.startsWith("gemini"))
            return "google";
        return "anthropic";
    }
}
/**
 * Global runner instance
 */
let globalRunner = null;
/**
 * Get or create global runner
 */
export function getRunner(config) {
    if (!globalRunner) {
        globalRunner = new MultiModelRunner(config);
    }
    return globalRunner;
}
/**
 * Reset global runner
 */
export function resetRunner() {
    globalRunner = null;
}
/**
 * Format provider name for display
 */
export function formatProvider(provider) {
    switch (provider) {
        case "anthropic":
            return "Anthropic";
        case "openai":
            return "OpenAI";
        case "google":
            return "Google";
        default:
            return provider;
    }
}
//# sourceMappingURL=runner.js.map