/**
 * Extended Benchmark Harness
 *
 * Provides percentile calculation utilities and named benchmark runners for
 * each major Nexus Prime engine. Results include p50/p95/p99 latency, throughput,
 * and peak RSS — shaped to match the investor benchmark report format.
 *
 * Market comparison baselines (from docs/investor/benchmark-report.md):
 *   Memory store:   Mem0 cloud ~50-200ms,  Zep ~30-100ms  → target: < 5ms
 *   Memory recall:  Zep vector ~30-100ms               → target: < 10ms
 *   Token ratio:    naive 1x, vLLM prefix ~2x           → target: ≥ 2x
 *   N-gram query:   full-scan ~O(N)                     → target: < 5ms
 *   Knowledge fab:  no direct competitor                → target: < 50ms
 *   Dashboard API:  –                                   → p95 < 300ms
 */
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
// ─────────────────────────────────────────────────────────────────────────────
// Utilities
// ─────────────────────────────────────────────────────────────────────────────
export function percentile(sortedSamples, p) {
    if (sortedSamples.length === 0)
        return 0;
    const idx = Math.min(Math.floor((p / 100) * sortedSamples.length), sortedSamples.length - 1);
    return sortedSamples[idx];
}
function sorted(samples) {
    return [...samples].sort((a, b) => a - b);
}
function rssBytes() {
    return process.memoryUsage().rss;
}
async function runIterations(fn, iterations) {
    const samples = [];
    for (let i = 0; i < iterations; i++) {
        const t0 = performance.now();
        await fn();
        samples.push(performance.now() - t0);
    }
    return samples;
}
function buildResult(name, samples, extra) {
    const s = sorted(samples);
    const totalMs = s.reduce((a, b) => a + b, 0);
    return {
        name,
        p50: percentile(s, 50),
        p95: percentile(s, 95),
        p99: percentile(s, 99),
        throughputPerSec: totalMs > 0 ? (s.length / totalMs) * 1000 : 0,
        peakRssBytes: rssBytes(),
        iterations: s.length,
        extra,
    };
}
// ─────────────────────────────────────────────────────────────────────────────
// Token Optimization Benchmark
// ─────────────────────────────────────────────────────────────────────────────
export async function runTokenOptimizationBenchmark(options) {
    const { TokenSupremacyEngine } = await import('./token-supremacy.js');
    const tmpDir = options.stateDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'nexus-bench-tokens-'));
    const srcDir = path.join(tmpDir, 'src');
    fs.mkdirSync(srcDir, { recursive: true });
    // Create a realistic file set (30 files, ~300 LOC each)
    const domains = ['auth', 'database', 'api', 'middleware', 'utils', 'logger', 'config',
        'types', 'server', 'client', 'cache', 'queue', 'jobs', 'models', 'routes',
        'controllers', 'services', 'helpers', 'validators', 'transformers'];
    for (const d of domains) {
        const content = [
            `// ${d}.ts — auto-generated benchmark fixture`,
            `export function ${d}Main(input: string): string { return input; }`,
            `export function ${d}Helper(x: number): number { return x * 2; }`,
            `export function ${d}Validate(v: unknown): boolean { return !!v; }`,
        ].join('\n').repeat(15) + '\n';
        fs.writeFileSync(path.join(srcDir, `${d}.ts`), content, 'utf8');
    }
    const files = fs.readdirSync(srcDir).map((f) => {
        const abs = path.join(srcDir, f);
        const stat = fs.statSync(abs);
        return { path: abs, sizeBytes: stat.size, lastModified: stat.mtimeMs };
    });
    const engine = new TokenSupremacyEngine(200_000);
    const naiveTokens = files.reduce((s, f) => s + Math.ceil(f.sizeBytes / 4), 0);
    const iter = options.iterations ?? 50;
    const tasks = ['fix authentication bug', 'optimize database queries', 'improve API performance',
        'refactor middleware stack', 'add logging to server'];
    let totalPlanTokens = 0;
    let planCount = 0;
    const samples = await runIterations(async () => {
        const task = tasks[planCount % tasks.length];
        const plan = await engine.plan(task, files);
        totalPlanTokens += plan.totalEstimatedTokens;
        planCount++;
    }, iter);
    const avgPlanTokens = totalPlanTokens / planCount;
    const compressionRatio = naiveTokens > 0 ? naiveTokens / Math.max(avgPlanTokens, 1) : 1;
    return buildResult('token_optimization', samples, {
        compressionRatio: Math.round(compressionRatio * 100) / 100,
        naiveTokens,
        avgPlanTokens: Math.round(avgPlanTokens),
        fileCount: files.length,
        marketBaseline_vllm: 2,
        marketBaseline_naive: 1,
    });
}
// ─────────────────────────────────────────────────────────────────────────────
// Memory Store / Recall Benchmark
// ─────────────────────────────────────────────────────────────────────────────
export async function runMemoryStoreBenchmark(options) {
    const { createMemoryEngine } = await import('./memory.js');
    const stateDir = options.stateDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'nexus-bench-mem-'));
    const dbPath = path.join(stateDir, 'bench.db');
    const memory = createMemoryEngine(dbPath);
    const iter = options.iterations ?? 100;
    let i = 0;
    const samples = await runIterations(() => {
        memory.store(`Benchmark entry ${i}: implementing authentication with JWT tokens and refresh flow`, 0.8, ['#benchmark', '#auth']);
        i++;
    }, iter);
    memory.close?.();
    return buildResult('memory_store', samples, {
        marketBaseline_mem0_ms: 100,
        marketBaseline_zep_ms: 65,
        target_ms: 5,
    });
}
export async function runMemoryRecallBenchmark(options) {
    const { createMemoryEngine } = await import('./memory.js');
    const stateDir = options.stateDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'nexus-bench-recall-'));
    const dbPath = path.join(stateDir, 'bench.db');
    const memory = createMemoryEngine(dbPath);
    const seedCount = options.seedCount ?? 100;
    const topics = ['auth', 'database', 'api', 'deployment', 'testing', 'security', 'cache', 'queue'];
    for (let i = 0; i < seedCount; i++) {
        const topic = topics[i % topics.length];
        memory.store(`${topic} implementation detail #${i}: key facts about ${topic} patterns`, 0.7, [`#${topic}`]);
    }
    const queries = ['authentication JWT', 'database query', 'api endpoint', 'deployment pipeline', 'cache strategy'];
    let qi = 0;
    const iter = options.iterations ?? 100;
    const samples = await runIterations(async () => {
        await memory.recall(queries[qi % queries.length], 5);
        qi++;
    }, iter);
    memory.close?.();
    return buildResult('memory_recall', samples, {
        seedItems: seedCount,
        marketBaseline_zep_ms: 65,
        marketBaseline_mem0_ms: 125,
        target_ms: 10,
    });
}
// ─────────────────────────────────────────────────────────────────────────────
// N-gram Index Benchmark
// ─────────────────────────────────────────────────────────────────────────────
export async function runNgramIndexBenchmark(options) {
    const { NgramIndex } = await import('./ngram-index.js');
    const stateDir = options.stateDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'nexus-bench-ngram-'));
    process.env.NEXUS_STATE_DIR = stateDir;
    const dbPath = path.join(stateDir, 'ngram.db');
    const index = new NgramIndex(dbPath);
    const docCount = options.docCount ?? 1000;
    // Index documents
    const t0 = performance.now();
    for (let i = 0; i < docCount; i++) {
        const topic = ['authentication', 'database', 'middleware', 'deployment', 'testing'][i % 5];
        index.addDocument(`doc_${i}`, `${topic} implementation using TypeScript and Node.js pattern ${i}`);
    }
    const indexTimeMs = performance.now() - t0;
    const queries = ['authentication middleware', 'database query', 'TypeScript pattern', 'deployment pipeline', 'testing framework'];
    let qi = 0;
    const iter = options.iterations ?? 100;
    const samples = await runIterations(() => {
        index.search(queries[qi % queries.length], 20);
        qi++;
    }, iter);
    return buildResult('ngram_index_query', samples, {
        docsIndexed: docCount,
        indexTimeMs: Math.round(indexTimeMs),
        indexTimePerDocMs: Math.round((indexTimeMs / docCount) * 100) / 100,
        target_ms: 5,
    });
}
// ─────────────────────────────────────────────────────────────────────────────
// Knowledge Fabric Benchmark
// ─────────────────────────────────────────────────────────────────────────────
export async function runKnowledgeFabricBenchmark(options) {
    const { KnowledgeFabricEngine } = await import('./knowledge-fabric.js');
    const stateDir = options.stateDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'nexus-bench-kf-'));
    process.env.NEXUS_STATE_DIR = stateDir;
    const engine = new KnowledgeFabricEngine({ stateRoot: stateDir });
    const tasks = [
        'fix authentication bug in login flow',
        'optimize database query performance',
        'add API rate limiting middleware',
        'improve test coverage for core engine',
        'deploy to production with zero downtime',
    ];
    let ti = 0;
    const iter = options.iterations ?? 30;
    const samples = await runIterations(() => {
        engine.compose({
            runtimeId: 'bench',
            sessionId: `bench-${ti}`,
            task: tasks[ti % tasks.length],
            candidateFiles: [],
            memoryMatches: [],
        });
        ti++;
    }, iter);
    return buildResult('knowledge_fabric_compose', samples, {
        target_ms: 50,
    });
}
// ─────────────────────────────────────────────────────────────────────────────
// Dashboard API Benchmark
// ─────────────────────────────────────────────────────────────────────────────
export async function runDashboardApiBenchmark(options) {
    const endpoints = [
        '/api/health',
        '/api/license',
        '/api/runtimes',
        '/api/synapse/health',
        '/api/architects/dispatch',
        '/api/architects/escalations',
        '/api/dashboard/summary',
        '/api/dashboard/surface/operate',
        '/api/dashboard/surface/memory',
        '/api/dashboard/surface/runs',
    ];
    // Warmup: 3 rounds to allow server-side caches to prime
    for (let w = 0; w < 3; w++) {
        await fetch(options.baseUrl + '/api/health').then((r) => r.json()).catch(() => { });
        await fetch(options.baseUrl + '/api/dashboard/surface/operate').then((r) => r.json()).catch(() => { });
    }
    const iter = options.iterations ?? 30;
    let ei = 0;
    const samples = await runIterations(async () => {
        const url = options.baseUrl + endpoints[ei % endpoints.length];
        const res = await fetch(url);
        await res.json();
        ei++;
    }, iter);
    return buildResult('dashboard_api', samples, {
        endpointsCovered: endpoints.length,
        target_p95_ms: 300,
    });
}
export async function runRecallQualityBenchmark(options) {
    const { createMemoryEngine } = await import('./memory.js');
    const stateDir = options.stateDir ?? fs.mkdtempSync(path.join(os.tmpdir(), 'nexus-bench-rq-'));
    const dbPath = path.join(stateDir, 'recall-quality.db');
    const memory = createMemoryEngine(dbPath);
    // Ground-truth corpus: each entry has a unique discriminator phrase and the
    // query that should surface it.  Queries use varied vocabulary (including
    // synonyms and paraphrases) to test semantic understanding beyond keyword match.
    const corpus = [
        { id: 'jwt-refresh', query: 'JWT token refresh strategy', content: 'JWT refresh: store refresh token in HttpOnly cookie, access token in memory; rotate on every use; revoke on logout via denylist' },
        { id: 'db-index', query: 'database index optimization', content: 'Compound index on (user_id, created_at) reduced query time from 420ms to 3ms on the events table' },
        { id: 'rate-limit', query: 'API rate limiting implementation', content: 'Token-bucket rate limiter: 100 req/min per IP, 1000 req/min per authenticated user, Redis sliding window' },
        { id: 'ghost-pass', query: 'ghost pass worktree safety mechanism', content: 'Ghost-pass creates isolated git worktree before mutations; dry-run diff shown for approval; merge only on explicit confirm' },
        { id: 'rag-chunk', query: 'RAG chunking strategy for code', content: 'Code chunking: split at function boundaries, 512-token max, 64-token overlap; semantic headers prepended for retrieval context' },
        { id: 'darwin', query: 'Darwin self-improvement loop', content: 'Darwin loop: hypothesis generated post-orchestration → run in sandbox → if improvement > 3% promote; max 2 auto-promotes/day' },
        { id: 'sse-reconnect', query: 'SSE event stream reconnect backoff', content: 'SSE reconnect: exponential backoff 1s → 30s cap, jitter ±200ms; last-event-id header sent on reconnect for replay' },
        { id: 'cas-dedup', query: 'content-addressable storage dedup', content: 'CAS deduplication: SHA-256 fingerprint stored on insert; duplicate check is O(1) index lookup, saves ~34% store calls in benchmarks' },
        { id: 'synapse-budget', query: 'Synapse operative token budget', content: 'Each Synapse operative gets token budget proportional to mission complexity score (0–1), capped at 50k tokens by default' },
        { id: 'temporal-graph', query: 'temporal knowledge graph valid_from', content: 'Temporal edges have valid_from/valid_to timestamps; querying "state at time T" filters edges where valid_from <= T AND (valid_to IS NULL OR valid_to > T)' },
        { id: 'bm25-recall', query: 'BM25 full text search scoring', content: 'Memory recall uses BM25 over SQLite FTS5 with k1=1.5 b=0.75; trigram re-rank applied to top-20 BM25 results for precision boost' },
        { id: 'phantom-merge', query: 'phantom worker merge oracle strategy', content: 'Phantom merge oracle uses git merge-tree for conflict preview; auto-merges non-overlapping hunks, escalates conflicts to operator' },
        { id: 'mcp-handshake', query: 'MCP tool registration and handshake', content: 'MCP tools registered at stdio startup; bootstrap hash sent as first event; client re-registers on hash mismatch to force fresh tool list' },
        { id: 'canvas-texture', query: 'dashboard canvas noise texture', content: 'Dashboard uses SVG fractalNoise filter at 0.035 opacity + 24px dot-grid for depth; no external image assets, pure CSS' },
        { id: 'zettel-link', query: 'Zettelkasten memory link formation', content: 'Zettelkasten links formed when cosine similarity > 0.72 between embeddings; max 10 links per node to bound graph density' },
        // ── Semantic/synonym queries — vocabulary mismatch tests ──────────────
        { id: 'auth-cookie', query: 'how does login session persistence work', content: 'Session persistence uses signed cookies with SameSite=Strict; CSRF token embedded in hidden form field; session TTL 24h with sliding window renewal' },
        { id: 'error-retry', query: 'handling transient failures gracefully', content: 'Retry policy for external service calls: 3 attempts, exponential backoff starting at 200ms, circuit breaker trips after 5 consecutive failures' },
        { id: 'mem-promote', query: 'how do memories get upgraded to long-term', content: 'Memory promotion: items accessed 3+ times within 48h move from prefrontal to hippocampus; items with priority > 0.9 and trust > 0.8 promote to cortex' },
        { id: 'build-cache', query: 'making builds faster with intermediate caching', content: 'TypeScript incremental compilation stores .tsbuildinfo; Turborepo hashes inputs to skip unchanged packages; average cold build 45s, warm build 8s' },
        { id: 'secret-mgmt', query: 'keeping credentials safe in the codebase', content: 'Secrets managed via .env files gitignored by default; production values injected at deploy from Vault; pre-commit hook scans for hardcoded API keys' },
    ];
    const seedCount = options.seedCount ?? corpus.length;
    const seed = corpus.slice(0, seedCount);
    // Store all items
    const storedIds = [];
    for (const item of seed) {
        const id = memory.store(item.content, 0.75, [`#${item.id}`, '#benchmark']);
        storedIds.push(typeof id === 'string' ? id : item.id);
    }
    // Pad corpus with distractor items to make retrieval non-trivial
    const distractorTopics = ['logging formatter', 'CI pipeline config', 'error boundary pattern',
        'CSS grid layout', 'websocket ping interval', 'linting rules', 'changelog format',
        'docker layer caching', 'env variable naming', 'npm audit policy'];
    for (let d = 0; d < Math.min(seed.length * 4, 60); d++) {
        const t = distractorTopics[d % distractorTopics.length];
        memory.store(`${t} detail ${d}: standard approach for ${t} in production Node.js applications`, 0.5, [`#distractor-${d}`]);
    }
    // Issue queries and check whether ground-truth appears in top-k
    let hits1 = 0, hits5 = 0, hits10 = 0;
    const samples = [];
    for (const item of seed) {
        const t0 = performance.now();
        const results = await memory.recall(item.query, 10);
        samples.push(performance.now() - t0);
        const resultContents = results
            .map((r) => (typeof r === 'string' ? r : (r.content ?? r.text ?? r.summary ?? '')));
        // Ground truth: result contains the unique discriminator phrase from the content
        const discriminator = item.content.substring(0, 40).toLowerCase();
        const rank = resultContents.findIndex((c) => c.toLowerCase().includes(discriminator));
        if (rank === 0)
            hits1++;
        if (rank >= 0 && rank < 5)
            hits5++;
        if (rank >= 0 && rank < 10)
            hits10++;
    }
    memory.close?.();
    const r1 = hits1 / seed.length;
    const r5 = hits5 / seed.length;
    const r10 = hits10 / seed.length;
    const embedMode = process.env.NEXUS_EMBED_MODE ?? 'local';
    const base = buildResult('recall_quality', samples, {
        r1: Math.round(r1 * 1000) / 10, // percentage
        r5: Math.round(r5 * 1000) / 10,
        r10: Math.round(r10 * 1000) / 10,
        seedItems: seed.length,
        distractors: Math.min(seed.length * 4, 60),
        embeddingMode: embedMode,
        marketBaseline_mempalace_r5: 96.6,
        marketBaseline_mempalace_r1: 82.4,
    });
    return { ...base, r1, r5, r10, totalQueries: seed.length, seedItems: seed.length };
}
// ─────────────────────────────────────────────────────────────────────────────
// Bootstrap Token Cost Benchmark
// ─────────────────────────────────────────────────────────────────────────────
//
// Measures the token footprint of a Nexus Prime bootstrap session — i.e. how
// many tokens worth of context a client agent receives before any task begins.
// Analogous to MemPalace's published "wake-up cost" of ~600–900 tokens.
export async function runBootstrapCostBenchmark(options) {
    const stateDir = options.stateDir ?? process.cwd();
    // Primary source: compiled instruction packet (the actual agent context brief)
    const packetPath = path.join(stateDir, '.agent', 'runtime', 'packet.json');
    let packetTokens = 0;
    let packetChars = 0;
    if (fs.existsSync(packetPath)) {
        const raw = fs.readFileSync(packetPath, 'utf8');
        packetChars = raw.length;
        packetTokens = Math.ceil(raw.length / 4); // ~4 chars/token heuristic
    }
    // Secondary: CLAUDE.md / AGENTS.md injected into every session
    const contextFiles = ['CLAUDE.md', 'AGENTS.md', '.agent/README.md'];
    let contextChars = 0;
    for (const f of contextFiles) {
        const p = path.join(stateDir, f);
        if (fs.existsSync(p))
            contextChars += fs.readFileSync(p, 'utf8').length;
    }
    const contextTokens = Math.ceil(contextChars / 4);
    const totalTokens = packetTokens + contextTokens;
    // Time the packet read (simulates bootstrap I/O cost)
    const samples = [];
    for (let i = 0; i < 20; i++) {
        const t0 = performance.now();
        if (fs.existsSync(packetPath))
            fs.readFileSync(packetPath, 'utf8');
        for (const f of contextFiles) {
            const p = path.join(stateDir, f);
            if (fs.existsSync(p))
                fs.readFileSync(p, 'utf8');
        }
        samples.push(performance.now() - t0);
    }
    return buildResult('bootstrap_token_cost', samples, {
        packetTokens,
        contextTokens,
        totalTokens,
        packetChars,
        contextChars,
        marketBaseline_mempalace_tokens: 750, // MemPalace published ~600-900
        marketNote: 'Nexus loads compiled packet + rules; MemPalace loads 4-layer hierarchy',
    });
}
export function buildReport(results, nexusVersion) {
    const failures = [];
    for (const r of results) {
        const e = r.extra ?? {};
        if (r.name === 'token_optimization' && typeof e.compressionRatio === 'number' && e.compressionRatio < 2) {
            failures.push(`token_optimization: compressionRatio ${e.compressionRatio}x < 2x target`);
        }
        if (r.name === 'memory_store' && r.p50 > 5) {
            failures.push(`memory_store: p50 ${r.p50.toFixed(2)}ms > 5ms target`);
        }
        if (r.name === 'memory_recall' && r.p50 > 10) {
            failures.push(`memory_recall: p50 ${r.p50.toFixed(2)}ms > 10ms target`);
        }
        if (r.name === 'ngram_index_query' && r.p50 > 5) {
            failures.push(`ngram_index_query: p50 ${r.p50.toFixed(2)}ms > 5ms target`);
        }
        if (r.name === 'knowledge_fabric_compose' && r.p50 > 50) {
            failures.push(`knowledge_fabric_compose: p50 ${r.p50.toFixed(2)}ms > 50ms target`);
        }
        if (r.name === 'dashboard_api' && r.p95 > 300) {
            failures.push(`dashboard_api: p95 ${r.p95.toFixed(2)}ms > 300ms target`);
        }
    }
    return {
        generatedAt: new Date().toISOString(),
        nexusVersion,
        results,
        summary: {
            allTargetsMet: failures.length === 0,
            failures,
        },
    };
}
