/**
 * Benchmark Suite for LLM Performance
 *
 * Comprehensive benchmarking for:
 * - TTFT (Time To First Token)
 * - TPS (Tokens Per Second)
 * - Latency percentiles (p50, p95, p99)
 * - VRAM usage
 * - Cost per request
 * - Cache hit rates
 */
import { EventEmitter } from 'events';
export interface BenchmarkConfig {
    /** Number of warmup runs before actual benchmark */
    warmupRuns?: number;
    /** Number of benchmark runs */
    runs?: number;
    /** Concurrent requests (for throughput testing) */
    concurrency?: number;
    /** Timeout per request in ms */
    timeout?: number;
    /** Whether to include VRAM monitoring */
    monitorVRAM?: boolean;
    /** Test prompts to use */
    prompts?: BenchmarkPrompt[];
}
export interface BenchmarkPrompt {
    name: string;
    prompt: string;
    expectedTokens?: number;
    category?: 'simple' | 'moderate' | 'complex' | 'code' | 'reasoning';
}
export interface BenchmarkRun {
    promptName: string;
    startTime: number;
    endTime: number;
    ttft: number;
    totalTime: number;
    inputTokens: number;
    outputTokens: number;
    tps: number;
    cost: number;
    success: boolean;
    error?: string;
    vramUsage?: number;
}
export interface BenchmarkResults {
    model: string;
    timestamp: Date;
    config: BenchmarkConfig;
    runs: BenchmarkRun[];
    summary: BenchmarkSummary;
}
export interface BenchmarkSummary {
    totalRuns: number;
    successfulRuns: number;
    failedRuns: number;
    ttft: PercentileStats;
    totalTime: PercentileStats;
    tps: PercentileStats;
    inputTokens: {
        total: number;
        average: number;
    };
    outputTokens: {
        total: number;
        average: number;
    };
    cost: {
        total: number;
        average: number;
    };
    throughput: number;
    vram?: {
        min: number;
        max: number;
        average: number;
    };
}
export interface PercentileStats {
    min: number;
    max: number;
    avg: number;
    p50: number;
    p95: number;
    p99: number;
    stdDev: number;
}
export type BenchmarkCallback = (prompt: string, onFirstToken?: () => void) => Promise<{
    content: string;
    inputTokens?: number;
    outputTokens?: number;
}>;
export declare const DEFAULT_PROMPTS: BenchmarkPrompt[];
export declare class BenchmarkSuite extends EventEmitter {
    private config;
    private runs;
    private isRunning;
    constructor(config?: BenchmarkConfig);
    /**
     * Run the benchmark suite
     */
    run(model: string, callback: BenchmarkCallback): Promise<BenchmarkResults>;
    /**
     * Run benchmarks sequentially
     */
    private runSequential;
    /**
     * Run benchmarks concurrently
     */
    private runConcurrent;
    /**
     * Execute a single benchmark run
     */
    private executeRun;
    /**
     * Calculate summary statistics
     */
    private calculateSummary;
    /**
     * Calculate percentile statistics
     */
    private calculatePercentiles;
    /**
     * Calculate a specific percentile
     */
    private percentile;
    /**
     * Format results for display
     */
    formatResults(results: BenchmarkResults): string;
    /**
     * Export results to JSON
     */
    exportJSON(results: BenchmarkResults): string;
    /**
     * Compare two benchmark results
     */
    compare(baseline: BenchmarkResults, current: BenchmarkResults): BenchmarkComparison;
    private percentDiff;
    /**
     * Get current config
     */
    getConfig(): Required<BenchmarkConfig>;
    /**
     * Update config
     */
    updateConfig(config: Partial<BenchmarkConfig>): void;
}
export interface BenchmarkComparison {
    baseline: string;
    current: string;
    ttft: {
        baseline: number;
        current: number;
        diff: number;
        improved: boolean;
    };
    tps: {
        baseline: number;
        current: number;
        diff: number;
        improved: boolean;
    };
    cost: {
        baseline: number;
        current: number;
        diff: number;
        improved: boolean;
    };
}
export declare function getBenchmarkSuite(config?: BenchmarkConfig): BenchmarkSuite;
export declare function resetBenchmarkSuite(): void;
