/**
 * Model Integration Benchmark
 *
 * Runs real API calls against multiple LLM providers via Factory.ai droid exec CLI
 * to compare model performance on UAP memory-enhanced tasks.
 *
 * Assumptions:
 * - FACTORY_API_KEY is set in environment for Factory.ai API access
 * - Models: Claude Opus 4.5, GLM 4.7, GPT 5.2
 * - droid CLI is installed and accessible
 * - UAP CLI is available for memory initialization
 *
 * What this handles:
 * - Full UAP setup (init, analyze, generate, memory start, prepopulate)
 * - CLAUDE.md reading and context injection
 * - Real API calls to multiple LLM providers via droid exec
 * - Task execution comparison across models with/without UAP
 * - Performance metrics collection (latency, success, tokens)
 * - Result aggregation and reporting
 *
 * What this does NOT handle:
 * - Rate limiting (caller responsibility)
 * - Cost tracking (would require billing API)
 * - Streaming responses (uses completion mode)
 */
import { readFileSync, writeFileSync, existsSync } from 'fs';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { execSync } from 'child_process';
import { getMaxParallel } from '../utils/system-resources.js';
import { concurrentMap } from '../utils/concurrency-pool.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const PROJECT_ROOT = join(__dirname, '../..');
/**
 * Initialize UAP system for benchmark testing
 * Runs: uam init, uam analyze, uam generate, uam memory start, uam memory prepopulate
 */
async function setupUAP(verbose = false) {
    const result = {
        initialized: false,
        memoryStarted: false,
        memoryPrepopulated: false,
        claudeMdLoaded: false,
        errors: [],
    };
    const log = (msg) => {
        if (verbose)
            console.log(`  [UAP Setup] ${msg}`);
    };
    try {
        // Step 1: Check if UAP CLI is available
        log('Checking UAP CLI availability...');
        try {
            execSync('uam --version', { encoding: 'utf-8', cwd: PROJECT_ROOT, stdio: 'pipe' });
        }
        catch {
            // Try with npx
            execSync('npx uam --version', { encoding: 'utf-8', cwd: PROJECT_ROOT, stdio: 'pipe' });
        }
        // Step 2: Initialize UAP (idempotent - safe to run multiple times)
        log('Running uam init...');
        try {
            execSync('uam init --non-interactive 2>/dev/null || true', {
                encoding: 'utf-8',
                cwd: PROJECT_ROOT,
                stdio: 'pipe',
                timeout: 30000,
            });
            result.initialized = true;
        }
        catch (e) {
            result.errors.push(`init failed: ${e instanceof Error ? e.message : String(e)}`);
        }
        // Step 3: Analyze project structure
        log('Running uam analyze...');
        try {
            execSync('uam analyze 2>/dev/null || true', {
                encoding: 'utf-8',
                cwd: PROJECT_ROOT,
                stdio: 'pipe',
                timeout: 60000,
            });
        }
        catch (e) {
            result.errors.push(`analyze failed: ${e instanceof Error ? e.message : String(e)}`);
        }
        // Step 4: Generate/update CLAUDE.md
        log('Running uam generate...');
        try {
            execSync('uam generate 2>/dev/null || true', {
                encoding: 'utf-8',
                cwd: PROJECT_ROOT,
                stdio: 'pipe',
                timeout: 30000,
            });
        }
        catch (e) {
            result.errors.push(`generate failed: ${e instanceof Error ? e.message : String(e)}`);
        }
        // Step 5: Start memory services
        log('Starting memory services...');
        try {
            execSync('uam memory start 2>/dev/null || true', {
                encoding: 'utf-8',
                cwd: PROJECT_ROOT,
                stdio: 'pipe',
                timeout: 60000,
            });
            result.memoryStarted = true;
        }
        catch (e) {
            result.errors.push(`memory start failed: ${e instanceof Error ? e.message : String(e)}`);
        }
        // Step 6: Prepopulate memory from docs and git history
        log('Prepopulating memory from docs and git...');
        try {
            execSync('uam memory prepopulate --docs --git --limit 100 2>/dev/null || true', {
                encoding: 'utf-8',
                cwd: PROJECT_ROOT,
                stdio: 'pipe',
                timeout: 120000,
            });
            result.memoryPrepopulated = true;
        }
        catch (e) {
            result.errors.push(`memory prepopulate failed: ${e instanceof Error ? e.message : String(e)}`);
        }
        // Step 7: Verify CLAUDE.md exists
        const claudeMdPath = join(PROJECT_ROOT, 'CLAUDE.md');
        if (existsSync(claudeMdPath)) {
            result.claudeMdLoaded = true;
            log('CLAUDE.md found and ready');
        }
        else {
            result.errors.push('CLAUDE.md not found after setup');
        }
    }
    catch (error) {
        result.errors.push(`UAP setup error: ${error instanceof Error ? error.message : String(error)}`);
    }
    return result;
}
/**
 * Load UAP memory context from CLAUDE.md and short-term memory
 */
function loadUAPMemoryContext() {
    const sections = [];
    // Read CLAUDE.md
    const claudeMdPath = join(PROJECT_ROOT, 'CLAUDE.md');
    if (existsSync(claudeMdPath)) {
        const claudeMd = readFileSync(claudeMdPath, 'utf-8');
        // Extract key sections from CLAUDE.md
        sections.push('## UAP Memory Context (from CLAUDE.md)\n');
        // Extract Code Field section
        const codeFieldMatch = claudeMd.match(/## .*CODE FIELD.*?(?=\n## |\n---\n|$)/s);
        if (codeFieldMatch) {
            sections.push('### Code Field Guidelines\n');
            sections.push(codeFieldMatch[0].slice(0, 1500) + '\n');
        }
        // Extract Testing Requirements
        const testingMatch = claudeMd.match(/## .*Testing Requirements.*?(?=\n## |\n---\n|$)/s);
        if (testingMatch) {
            sections.push('### Testing Requirements\n');
            sections.push(testingMatch[0].slice(0, 500) + '\n');
        }
        // Extract Repository Structure
        const structureMatch = claudeMd.match(/## Repository Structure.*?```[\s\S]*?```/);
        if (structureMatch) {
            sections.push('### Repository Structure\n');
            sections.push(structureMatch[0].slice(0, 1000) + '\n');
        }
    }
    // Query short-term memory from SQLite
    const dbPath = join(PROJECT_ROOT, 'agents/data/memory/short_term.db');
    if (existsSync(dbPath)) {
        try {
            const recentMemories = execSync(`sqlite3 "${dbPath}" "SELECT type, content FROM memories ORDER BY id DESC LIMIT 10;" 2>/dev/null || true`, { encoding: 'utf-8', cwd: PROJECT_ROOT }).trim();
            if (recentMemories) {
                sections.push('### Recent Session Memory\n');
                sections.push('```\n' + recentMemories.slice(0, 1000) + '\n```\n');
            }
            // Get lessons learned
            const lessons = execSync(`sqlite3 "${dbPath}" "SELECT content FROM memories WHERE type='lesson' ORDER BY id DESC LIMIT 5;" 2>/dev/null || true`, { encoding: 'utf-8', cwd: PROJECT_ROOT }).trim();
            if (lessons) {
                sections.push('### Lessons Learned\n');
                sections.push(lessons.slice(0, 500) + '\n');
            }
        }
        catch {
            // Memory DB not available
        }
    }
    // Add static context as fallback/supplement
    sections.push(`
### Project Coding Standards
- Use TypeScript strict mode
- All functions must have JSDoc comments with @param and @returns
- Error handling uses custom AppError class that extends Error
- Prefer async/await over callbacks and Promises
- Use zod for runtime input validation
- Export types and interfaces alongside implementations
- Use Map for key-value storage, Set for unique collections

### Common Patterns
- Singleton pattern: private constructor + static getInstance()
- Strategy pattern: interface + multiple implementations
- Factory pattern: static create() methods
- Error handling: try/catch with specific error types
- Exponential backoff: delay = baseMs * Math.pow(2, attempt)

### Known Gotchas (from memory)
- Always check array bounds: use i < length, not i <= length
- Handle empty arrays explicitly before operations
- Include cleanup logic for resources (timers, connections)
- JSON.parse throws on invalid input - always wrap in try/catch
- Array methods like reduce need initial value for empty arrays
- Map.get() returns undefined for missing keys

---

`);
    return sections.join('\n');
}
// Cached memory context (loaded once per benchmark run)
let cachedMemoryContext = null;
function getUAPMemoryContext() {
    if (!cachedMemoryContext) {
        cachedMemoryContext = loadUAPMemoryContext();
    }
    return cachedMemoryContext;
}
// ============================================================================
// Model Configurations (per Factory.ai droid CLI available models)
// ============================================================================
const MODELS = [
    {
        id: 'opus-4.6',
        name: 'Claude Opus 4.6',
        provider: 'anthropic',
        apiModel: 'claude-opus-4-6-20260101',
    },
    {
        id: 'sonnet-4.6',
        name: 'Claude Sonnet 4.6',
        provider: 'anthropic',
        apiModel: 'claude-sonnet-4-6-20250514',
    },
    {
        id: 'haiku',
        name: 'Claude Haiku (Latest)',
        provider: 'anthropic',
        apiModel: 'claude-3-5-haiku-20241022',
    },
    {
        id: 'gpt-5.3-codex',
        name: 'GPT 5.3 Codex',
        provider: 'openai',
        apiModel: 'gpt-5.3-codex',
    },
    {
        id: 'gpt-5.4',
        name: 'GPT 5.4',
        provider: 'openai',
        apiModel: 'gpt-5.4',
    },
    {
        id: 'qwen35-a3b',
        name: 'Qwen 3.5 35B A3B',
        provider: 'custom',
        apiModel: 'qwen35-a3b-iq4xs',
    },
];
// ============================================================================
// Benchmark Tasks
// ============================================================================
const BENCHMARK_TASKS = [
    {
        id: 'task-001-code-generation',
        name: 'TypeScript Function Generation',
        description: 'Generate a well-typed TypeScript function',
        prompt: `Write a TypeScript function called 'calculateAverage' that:
1. Takes an array of numbers as input
2. Returns the arithmetic mean
3. Handles empty arrays (return 0)
4. Has proper type annotations

Return ONLY the function code, no explanations.`,
        difficulty: 'easy',
        category: 'code-generation',
        expectedPatterns: ['function calculateAverage', 'number[]', ': number', 'length', 'return'],
        maxTokens: 500,
    },
    {
        id: 'task-002-bug-fix',
        name: 'Bug Detection and Fix',
        description: 'Identify and fix a bug in code',
        prompt: `Find and fix the bug in this TypeScript code:

function sumPositive(nums: number[]): number {
  let sum = 0;
  for (let i = 0; i <= nums.length; i++) {
    if (nums[i] > 0) {
      sum += nums[i];
    }
  }
  return sum;
}

Return ONLY the corrected function code.`,
        difficulty: 'easy',
        category: 'bug-fix',
        expectedPatterns: ['i < nums.length', 'function sumPositive', 'return sum'],
        maxTokens: 500,
    },
    {
        id: 'task-003-pattern-application',
        name: 'Design Pattern Implementation',
        description: 'Implement a singleton pattern',
        prompt: `Implement a TypeScript singleton class called 'ConfigManager' that:
1. Has a private constructor
2. Has a static getInstance() method
3. Has get(key: string) and set(key: string, value: any) methods
4. Stores configuration in a private Map

Return ONLY the class code.`,
        difficulty: 'medium',
        category: 'patterns',
        expectedPatterns: [
            'class ConfigManager',
            'private constructor',
            'static getInstance',
            'private static instance',
            'Map',
        ],
        maxTokens: 800,
    },
    {
        id: 'task-004-refactoring',
        name: 'Code Refactoring',
        description: 'Refactor code for better maintainability',
        prompt: `Refactor this code to follow SOLID principles and improve readability:

function processOrder(order: any) {
  if (order.type === 'digital') {
    console.log('Sending email with download link');
    order.status = 'delivered';
  } else if (order.type === 'physical') {
    console.log('Creating shipping label');
    order.status = 'shipped';
  } else if (order.type === 'subscription') {
    console.log('Activating subscription');
    order.status = 'active';
  }
  console.log('Order processed: ' + order.id);
  return order;
}

Provide the refactored TypeScript code using proper interfaces and a strategy pattern.`,
        difficulty: 'medium',
        category: 'refactoring',
        expectedPatterns: ['interface', 'class', 'implements', 'process'],
        maxTokens: 1200,
    },
    {
        id: 'task-005-memory-context',
        name: 'Context-Aware Code Generation',
        description: 'Generate code using provided context',
        prompt: `Given the following project context from memory:

MEMORY CONTEXT:
- Project uses src/utils/ for utility functions
- All functions must have JSDoc comments
- Error handling uses custom AppError class
- Prefer async/await over callbacks
- Use zod for input validation

Write a utility function 'validateAndParseJSON' that:
1. Takes a string input
2. Validates it's valid JSON using zod
3. Returns the parsed object or throws AppError
4. Has proper JSDoc documentation

Return ONLY the function code with JSDoc.`,
        difficulty: 'medium',
        category: 'memory',
        expectedPatterns: ['async', 'zod', 'AppError', '@param', '@returns', 'validateAndParseJSON'],
        maxTokens: 800,
    },
    {
        id: 'task-006-complex-algorithm',
        name: 'Algorithm Implementation',
        description: 'Implement a complex algorithm with proper typing',
        prompt: `Implement a TypeScript function 'findShortestPath' using Dijkstra's algorithm:

1. Input: weighted graph as adjacency list Map<string, Map<string, number>>
2. Input: start node (string), end node (string)
3. Output: { path: string[], distance: number } or null if no path
4. Handle disconnected nodes properly
5. Use proper TypeScript types

Return ONLY the function code with type definitions.`,
        difficulty: 'hard',
        category: 'algorithms',
        expectedPatterns: [
            'function findShortestPath',
            'Map<string',
            'distance',
            'path',
            'while',
            'return',
        ],
        maxTokens: 1500,
    },
    {
        id: 'task-007-multi-step-task',
        name: 'Multi-Step Code Generation',
        description: 'Complete a multi-step implementation task',
        prompt: `Create a complete TypeScript module for a rate limiter with these requirements:

1. Interface RateLimiterConfig { maxRequests: number; windowMs: number; }
2. Class RateLimiter with:
   - constructor(config: RateLimiterConfig)
   - isAllowed(clientId: string): boolean
   - getRemainingRequests(clientId: string): number
   - reset(clientId?: string): void
3. Use Map for tracking requests per client
4. Include proper cleanup of expired entries
5. Export both the class and interface

Return the complete module code.`,
        difficulty: 'hard',
        category: 'multi-step',
        expectedPatterns: [
            'interface RateLimiterConfig',
            'class RateLimiter',
            'isAllowed',
            'getRemainingRequests',
            'reset',
            'Map',
            'export',
        ],
        maxTokens: 2000,
    },
    {
        id: 'task-008-error-handling',
        name: 'Comprehensive Error Handling',
        description: 'Implement robust error handling',
        prompt: `Create a TypeScript async function 'fetchWithRetry' that:

1. Takes url: string, options?: RequestInit, retryConfig?: { maxRetries: number; backoffMs: number; }
2. Implements exponential backoff retry logic
3. Handles network errors, timeout, and HTTP errors (4xx, 5xx)
4. Returns Promise<Response> or throws a detailed custom error
5. Logs each retry attempt
6. Has proper TypeScript types for all parameters and return values

Return ONLY the function code with any necessary type definitions.`,
        difficulty: 'hard',
        category: 'error-handling',
        expectedPatterns: [
            'async function fetchWithRetry',
            'retry',
            'backoff',
            'catch',
            'throw',
            'Promise<Response>',
        ],
        maxTokens: 1200,
    },
];
// ============================================================================
// Droid Exec Client
// ============================================================================
class DroidExecClient {
    apiKey;
    tmpDir;
    autoLevel;
    constructor(apiKey, autoLevel = 'low') {
        this.apiKey = apiKey;
        this.autoLevel = autoLevel;
        this.tmpDir = '/tmp/uap-benchmark';
        try {
            execSync(`mkdir -p ${this.tmpDir}`, { encoding: 'utf-8' });
        }
        catch {
            // ignore
        }
    }
    async complete(model, prompt) {
        const startTime = Date.now();
        // Write prompt to temp file to avoid shell escaping issues
        const promptFile = `${this.tmpDir}/prompt-${Date.now()}.txt`;
        writeFileSync(promptFile, prompt, 'utf-8');
        try {
            // Use --auto low to allow file operations without system modifications
            const result = execSync(`FACTORY_API_KEY="${this.apiKey}" droid exec --model "${model}" --auto ${this.autoLevel} -f "${promptFile}"`, {
                encoding: 'utf-8',
                timeout: 300000, // 5 minutes for complex tasks
                maxBuffer: 10 * 1024 * 1024,
                env: { ...process.env, FACTORY_API_KEY: this.apiKey },
            });
            const latencyMs = Date.now() - startTime;
            // Clean up temp file
            try {
                execSync(`rm "${promptFile}"`, { encoding: 'utf-8' });
            }
            catch {
                // ignore cleanup failures
            }
            return {
                content: result.trim(),
                tokensUsed: 0,
                latencyMs,
            };
        }
        catch (error) {
            // Clean up temp file
            try {
                execSync(`rm "${promptFile}"`, { encoding: 'utf-8' });
            }
            catch {
                // ignore cleanup failures
            }
            const errMsg = error instanceof Error ? error.message : String(error);
            throw new Error(`droid exec failed: ${errMsg}`);
        }
    }
}
// ============================================================================
// Benchmark Runner
// ============================================================================
function evaluateResponse(response, expectedPatterns) {
    const normalizedResponse = response.toLowerCase();
    return expectedPatterns.filter((pattern) => normalizedResponse.includes(pattern.toLowerCase()));
}
async function runTaskForModel(client, model, task, withMemory = false) {
    const result = {
        taskId: task.id,
        modelId: model.id,
        success: false,
        latencyMs: 0,
        tokensUsed: 0,
        response: '',
        matchedPatterns: [],
    };
    try {
        // Inject UAP memory context if enabled (loaded from CLAUDE.md + memory DB)
        const prompt = withMemory ? getUAPMemoryContext() + task.prompt : task.prompt;
        const completion = await client.complete(model.apiModel, prompt);
        result.response = completion.content;
        result.latencyMs = completion.latencyMs;
        result.tokensUsed = completion.tokensUsed;
        result.matchedPatterns = evaluateResponse(completion.content, task.expectedPatterns);
        const matchRatio = result.matchedPatterns.length / task.expectedPatterns.length;
        result.success = matchRatio >= 0.6;
    }
    catch (error) {
        result.error = error instanceof Error ? error.message : String(error);
    }
    return result;
}
async function runBenchmarkForModel(client, model, tasks, withMemory = false) {
    const memoryLabel = withMemory ? ' (with UAP Memory)' : ' (without Memory)';
    console.log(`\n${'='.repeat(60)}`);
    console.log(`Running benchmark for: ${model.name}${memoryLabel}`);
    console.log(`${'='.repeat(60)}`);
    const results = [];
    for (const task of tasks) {
        console.log(`  [${task.difficulty.toUpperCase()}] ${task.name}...`);
        const result = await runTaskForModel(client, model, task, withMemory);
        results.push(result);
        if (result.success) {
            console.log(`    ✓ Success (${result.latencyMs}ms)`);
        }
        else {
            console.log(`    ✗ Failed: ${result.error || 'Pattern mismatch'}`);
        }
        // Small delay between tasks
        await new Promise((r) => setTimeout(r, 1000));
    }
    const succeeded = results.filter((r) => r.success).length;
    const successfulResults = results.filter((r) => r.latencyMs > 0);
    const avgLatency = successfulResults.length > 0
        ? successfulResults.reduce((sum, r) => sum + r.latencyMs, 0) / successfulResults.length
        : 0;
    const totalTokens = results.reduce((sum, r) => sum + r.tokensUsed, 0);
    return {
        modelId: model.id,
        modelName: model.name,
        tasksRun: tasks.length,
        tasksSucceeded: succeeded,
        successRate: (succeeded / tasks.length) * 100,
        avgLatencyMs: Math.round(avgLatency),
        totalTokens,
        results,
    };
}
function generateComparison(modelResults) {
    const sorted = [...modelResults].sort((a, b) => b.successRate - a.successRate);
    const fastest = [...modelResults].sort((a, b) => a.avgLatencyMs - b.avgLatencyMs);
    const byDifficulty = {};
    for (const diff of ['easy', 'medium', 'hard']) {
        let bestModel = '';
        let bestRate = 0;
        for (const modelResult of modelResults) {
            const diffTasks = modelResult.results.filter((r) => {
                const task = BENCHMARK_TASKS.find((t) => t.id === r.taskId);
                return task?.difficulty === diff;
            });
            if (diffTasks.length > 0) {
                const rate = (diffTasks.filter((t) => t.success).length / diffTasks.length) * 100;
                if (rate > bestRate) {
                    bestRate = rate;
                    bestModel = modelResult.modelName;
                }
            }
        }
        byDifficulty[diff] = { model: bestModel, successRate: bestRate };
    }
    return {
        bestOverall: sorted[0]?.modelName || 'N/A',
        fastestModel: fastest[0]?.modelName || 'N/A',
        mostAccurate: sorted[0]?.modelName || 'N/A',
        byDifficulty,
    };
}
function generateMarkdownReport(report) {
    const lines = [
        '# Model Integration Benchmark Results',
        '',
        `**Generated:** ${report.timestamp}`,
        `**Models Tested:** ${report.models.map((m) => m.modelName).join(', ')}`,
        `**Tasks Run:** ${BENCHMARK_TASKS.length}`,
        '',
        '---',
        '',
        '## Executive Summary',
        '',
        '| Model | Success Rate | Avg Latency | Total Tokens |',
        '|-------|--------------|-------------|--------------|',
    ];
    for (const model of report.models) {
        lines.push(`| ${model.modelName} | ${model.successRate.toFixed(1)}% | ${model.avgLatencyMs}ms | ${model.totalTokens} |`);
    }
    lines.push('', '---', '', '## Comparison', '');
    lines.push(`- **Best Overall:** ${report.comparison.bestOverall}`);
    lines.push(`- **Fastest Model:** ${report.comparison.fastestModel}`);
    lines.push(`- **Most Accurate:** ${report.comparison.mostAccurate}`);
    lines.push('', '### By Difficulty', '');
    lines.push('| Difficulty | Best Model | Success Rate |');
    lines.push('|------------|------------|--------------|');
    for (const [diff, data] of Object.entries(report.comparison.byDifficulty)) {
        lines.push(`| ${diff} | ${data.model} | ${data.successRate.toFixed(1)}% |`);
    }
    lines.push('', '---', '', '## Detailed Results', '');
    for (const model of report.models) {
        lines.push(`### ${model.modelName}`, '');
        lines.push('| Task | Difficulty | Success | Latency | Patterns Matched |');
        lines.push('|------|------------|---------|---------|------------------|');
        for (const result of model.results) {
            const task = BENCHMARK_TASKS.find((t) => t.id === result.taskId);
            const status = result.success ? '✓' : '✗';
            const patterns = `${result.matchedPatterns.length}/${task?.expectedPatterns.length || 0}`;
            lines.push(`| ${task?.name || result.taskId} | ${task?.difficulty || 'N/A'} | ${status} | ${result.latencyMs}ms | ${patterns} |`);
        }
        lines.push('');
    }
    // Add memory comparison section if available
    if (report.memoryComparison) {
        lines.push('---', '', '## UAP Memory Impact Analysis', '');
        lines.push('### Success Rate Comparison', '');
        lines.push('| Model | Without Memory | With Memory | Improvement |');
        lines.push('|-------|----------------|-------------|-------------|');
        for (const withMem of report.memoryComparison.withMemory) {
            const without = report.memoryComparison.withoutMemory.find((r) => r.modelId === withMem.modelId);
            const imp = report.memoryComparison.improvement[withMem.modelId];
            if (without && imp) {
                const sign = imp.successDelta >= 0 ? '+' : '';
                lines.push(`| ${withMem.modelName} | ${without.successRate.toFixed(1)}% | ${withMem.successRate.toFixed(1)}% | ${sign}${imp.successDelta.toFixed(1)}% |`);
            }
        }
        lines.push('', '### Latency Comparison', '');
        lines.push('| Model | Without Memory | With Memory | Speed Ratio |');
        lines.push('|-------|----------------|-------------|-------------|');
        for (const withMem of report.memoryComparison.withMemory) {
            const without = report.memoryComparison.withoutMemory.find((r) => r.modelId === withMem.modelId);
            const imp = report.memoryComparison.improvement[withMem.modelId];
            if (without && imp) {
                const speedLabel = imp.speedupRatio > 1
                    ? `${imp.speedupRatio.toFixed(2)}x faster`
                    : imp.speedupRatio < 1
                        ? `${(1 / imp.speedupRatio).toFixed(2)}x slower`
                        : 'same';
                lines.push(`| ${withMem.modelName} | ${without.avgLatencyMs}ms | ${withMem.avgLatencyMs}ms | ${speedLabel} |`);
            }
        }
        lines.push('', '### Key Findings', '');
        // Find best improvement
        const improvements = Object.entries(report.memoryComparison.improvement);
        if (improvements.length > 0) {
            const bestImprovement = improvements.reduce((a, b) => a[1].successDelta > b[1].successDelta ? a : b);
            const bestModel = BENCHMARK_TASKS.length > 0
                ? report.memoryComparison.withMemory.find((m) => m.modelId === bestImprovement[0])
                    ?.modelName
                : 'N/A';
            lines.push(`- **Best Memory Benefit:** ${bestModel} (+${bestImprovement[1].successDelta.toFixed(1)}% success rate)`);
            const avgImprovement = improvements.reduce((sum, [_, imp]) => sum + imp.successDelta, 0) / improvements.length;
            lines.push(`- **Average Improvement:** +${avgImprovement.toFixed(1)}% success rate across all models`);
            lines.push('', '### Interpretation', '');
            lines.push('UAP memory context injection provides models with:');
            lines.push('- Project structure knowledge (file locations, patterns)');
            lines.push('- Coding standards (JSDoc, error handling, async patterns)');
            lines.push('- Common gotchas and lessons learned from previous sessions');
            lines.push('- Design pattern templates (singleton, strategy, factory)');
        }
    }
    lines.push('', '---', '', '**Report Generated by UAP Model Integration Benchmark**');
    return lines.join('\n');
}
// ============================================================================
// Parallel Execution Utilities
// ============================================================================
/**
 * Run multiple model benchmarks in parallel with configurable concurrency.
 * Uses shared concurrentMap utility with auto-detected parallelism.
 */
async function runModelsInParallel(client, models, tasks, withMemory, concurrency) {
    return concurrentMap(models, async (model) => runBenchmarkForModel(client, model, tasks, withMemory), { maxConcurrent: concurrency });
}
export async function runModelBenchmark(apiKeyOrOptions, modelIds, compareMemory = true, parallelModels = 1) {
    // Handle both old signature and new options object
    let key;
    let models;
    let compare;
    let parallel;
    if (typeof apiKeyOrOptions === 'object' && apiKeyOrOptions !== null) {
        key = apiKeyOrOptions.apiKey;
        models = apiKeyOrOptions.modelIds;
        compare = apiKeyOrOptions.compareMemory ?? true;
        parallel = apiKeyOrOptions.parallelModels ?? 1;
    }
    else {
        key = apiKeyOrOptions;
        models = modelIds;
        compare = compareMemory;
        parallel = parallelModels || 1;
    }
    // Auto-detect parallelism from env or system resources
    if (parallel <= 1) {
        const envParallel = parseInt(process.env.UAP_BENCHMARK_PARALLEL || '', 10);
        if (envParallel > 0) {
            parallel = envParallel;
        }
        else {
            parallel = getMaxParallel('io');
        }
    }
    key = key || process.env.FACTORY_API_KEY || process.env.DROID_API_KEY;
    if (!key) {
        throw new Error('FACTORY_API_KEY or DROID_API_KEY not provided and not found in environment');
    }
    const client = new DroidExecClient(key, 'medium');
    const modelsToTest = models ? MODELS.filter((m) => models.includes(m.id)) : MODELS;
    if (modelsToTest.length === 0) {
        throw new Error('No valid models specified');
    }
    // Determine effective parallelism
    const effectiveParallel = Math.min(parallel, modelsToTest.length);
    const isParallel = effectiveParallel > 1;
    console.log('\n' + '='.repeat(60));
    console.log('   UAP MODEL INTEGRATION BENCHMARK');
    console.log('='.repeat(60));
    console.log(`\nModels: ${modelsToTest.map((m) => m.name).join(', ')}`);
    console.log(`Tasks: ${BENCHMARK_TASKS.length}`);
    console.log(`Memory Comparison: ${compare ? 'ENABLED' : 'DISABLED'}`);
    console.log(`Parallel Models: ${effectiveParallel}${isParallel ? ' (ENABLED)' : ' (sequential)'}`);
    let withoutMemoryResults = [];
    let withMemoryResults = [];
    // Run without memory first
    console.log('\n' + '█'.repeat(60));
    console.log(`   PHASE 1: WITHOUT UAP MEMORY${isParallel ? ' (PARALLEL)' : ''}`);
    console.log('█'.repeat(60));
    if (isParallel) {
        console.log(`\n  Running ${modelsToTest.length} models with concurrency=${effectiveParallel}...\n`);
        withoutMemoryResults = await runModelsInParallel(client, modelsToTest, BENCHMARK_TASKS, false, effectiveParallel);
    }
    else {
        for (const model of modelsToTest) {
            const result = await runBenchmarkForModel(client, model, BENCHMARK_TASKS, false);
            withoutMemoryResults.push(result);
        }
    }
    // Run with memory if comparison enabled
    if (compare) {
        console.log('\n' + '█'.repeat(60));
        console.log(`   PHASE 2: WITH UAP MEMORY${isParallel ? ' (PARALLEL)' : ''}`);
        console.log('█'.repeat(60));
        // Setup UAP before running with-memory tests
        console.log('\n--- Setting up UAP (init, analyze, generate, memory start, prepopulate) ---');
        const uapSetup = await setupUAP(true);
        if (uapSetup.errors.length > 0) {
            console.log('\nUAP Setup warnings:');
            uapSetup.errors.forEach((e) => console.log(`  - ${e}`));
        }
        console.log(`\nUAP Status:`);
        console.log(`  Initialized: ${uapSetup.initialized ? '✓' : '✗'}`);
        console.log(`  Memory Started: ${uapSetup.memoryStarted ? '✓' : '✗'}`);
        console.log(`  Memory Prepopulated: ${uapSetup.memoryPrepopulated ? '✓' : '✗'}`);
        console.log(`  CLAUDE.md Loaded: ${uapSetup.claudeMdLoaded ? '✓' : '✗'}`);
        // Clear cached context to force reload with fresh memory
        cachedMemoryContext = null;
        // Log memory context size
        const memoryContext = getUAPMemoryContext();
        console.log(`  Memory Context Size: ${memoryContext.length} chars\n`);
        if (isParallel) {
            console.log(`  Running ${modelsToTest.length} models with concurrency=${effectiveParallel}...\n`);
            withMemoryResults = await runModelsInParallel(client, modelsToTest, BENCHMARK_TASKS, true, effectiveParallel);
        }
        else {
            for (const model of modelsToTest) {
                const result = await runBenchmarkForModel(client, model, BENCHMARK_TASKS, true);
                withMemoryResults.push(result);
            }
        }
    }
    // Calculate memory improvement for each model
    const improvement = {};
    if (compare) {
        for (const model of modelsToTest) {
            const without = withoutMemoryResults.find((r) => r.modelId === model.id);
            const withMem = withMemoryResults.find((r) => r.modelId === model.id);
            if (without && withMem) {
                improvement[model.id] = {
                    successDelta: withMem.successRate - without.successRate,
                    speedupRatio: without.avgLatencyMs > 0 ? without.avgLatencyMs / withMem.avgLatencyMs : 1,
                };
            }
        }
    }
    // Use with-memory results as primary if available, otherwise without
    const primaryResults = compare && withMemoryResults.length > 0 ? withMemoryResults : withoutMemoryResults;
    const report = {
        timestamp: new Date().toISOString(),
        models: primaryResults,
        comparison: generateComparison(primaryResults),
        memoryComparison: compare
            ? {
                withMemory: withMemoryResults,
                withoutMemory: withoutMemoryResults,
                improvement,
            }
            : undefined,
    };
    // Generate and save markdown report
    const markdown = generateMarkdownReport(report);
    const reportPath = join(__dirname, '../../MODEL_BENCHMARK_RESULTS.md');
    writeFileSync(reportPath, markdown);
    console.log(`\nReport saved to: ${reportPath}`);
    // Print summary
    console.log('\n' + '='.repeat(60));
    console.log('   BENCHMARK COMPLETE');
    console.log('='.repeat(60));
    if (compare) {
        console.log('\n--- Without Memory ---');
        for (const model of withoutMemoryResults) {
            console.log(`  ${model.modelName}: ${model.successRate.toFixed(1)}% success, ${model.avgLatencyMs}ms avg`);
        }
        console.log('\n--- With UAP Memory ---');
        for (const model of withMemoryResults) {
            console.log(`  ${model.modelName}: ${model.successRate.toFixed(1)}% success, ${model.avgLatencyMs}ms avg`);
        }
        console.log('\n--- Memory Improvement ---');
        for (const [modelId, imp] of Object.entries(improvement)) {
            const model = modelsToTest.find((m) => m.id === modelId);
            const sign = imp.successDelta >= 0 ? '+' : '';
            console.log(`  ${model?.name ?? modelId}: ${sign}${imp.successDelta.toFixed(1)}% success, ${imp.speedupRatio.toFixed(2)}x speed`);
        }
    }
    else {
        console.log('\nSummary:');
        for (const model of primaryResults) {
            console.log(`  ${model.modelName}: ${model.successRate.toFixed(1)}% success, ${model.avgLatencyMs}ms avg`);
        }
    }
    console.log(`\nBest Overall: ${report.comparison.bestOverall}`);
    return report;
}
// CLI entry point
if (process.argv[1]?.includes('model-integration')) {
    const envPath = join(__dirname, '../../.env');
    if (existsSync(envPath)) {
        const envContent = readFileSync(envPath, 'utf-8');
        for (const line of envContent.split('\n')) {
            const [key, ...valueParts] = line.split('=');
            if (key && valueParts.length > 0) {
                process.env[key.trim()] = valueParts.join('=').trim();
            }
        }
    }
    runModelBenchmark()
        .then(() => process.exit(0))
        .catch((err) => {
        console.error('Benchmark failed:', err);
        process.exit(1);
    });
}
export { MODELS, BENCHMARK_TASKS, setupUAP, loadUAPMemoryContext };
//# sourceMappingURL=model-integration.js.map