// Copyright (c) 2026 defconxt. All rights reserved.
// Licensed under AGPL-3.0 — see LICENSE file for details.

/**
 * NERF Benchmark — Runner (orchestrates build → start → solve → score → report).
 *
 * Supports parallel execution with concurrency limits and retry/best-of-N
 * for reliability on flaky benchmarks.
 */

import { BenchmarkBuilder } from './builder.js';
import { HarnessConfig } from './config.js';
import { SolverResult } from './models.js';
import { aggregateResults, scoreResult } from './scorer.js';
import { getSolver } from './solver.js';

// ---------------------------------------------------------------------------
// Single benchmark execution
// ---------------------------------------------------------------------------

export async function runSingleBenchmark(builder, solver, name, flag) {
  const config = builder.getBenchmark(name);
  flag = flag || builder.config.generateFlag(name);
  let targetUrl = '';
  try {
    const result = builder.runBenchmark(name, flag);
    targetUrl = result.targetUrl;
    const solverResult = await solver.solve(config, targetUrl, result.expectedFlag);
    return scoreResult(config, solverResult, flag, targetUrl);
  } catch (e) {
    return scoreResult(config, new SolverResult({ error: String(e) }), flag, targetUrl);
  } finally {
    try { builder.stop(name); } catch { /* ignore */ }
  }
}

// ---------------------------------------------------------------------------
// Retry logic — best-of-N
// ---------------------------------------------------------------------------

/**
 * Run a single benchmark with retries, keeping the best result.
 *
 * @param {BenchmarkBuilder} builder
 * @param {object} solver
 * @param {string} name
 * @param {object} [opts]
 * @param {number} [opts.retries=1] - Total attempts (1 = no retry)
 * @param {string} [opts.flag]
 * @param {Function} [opts.onAttempt] - Called with (attemptNum, result) after each attempt
 * @returns {Promise<import('./models.js').BenchmarkResult>}
 */
export async function runWithRetry(builder, solver, name, opts = {}) {
  const maxAttempts = opts.retries ?? 1;
  const flag = opts.flag || builder.config.generateFlag(name);
  let bestResult = null;

  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
    const result = await runSingleBenchmark(builder, solver, name, flag);

    if (opts.onAttempt) opts.onAttempt(attempt, result);

    // Keep the best result: passed > failed > error
    if (!bestResult || _isBetterResult(result, bestResult)) {
      bestResult = result;
    }

    // If passed, no need to retry
    if (result.passed) break;
  }

  return bestResult;
}

/**
 * Compare two results — passed beats failed, failed beats error.
 * @param {import('./models.js').BenchmarkResult} a
 * @param {import('./models.js').BenchmarkResult} b
 * @returns {boolean} True if a is better than b
 */
function _isBetterResult(a, b) {
  if (a.passed && !b.passed) return true;
  if (!a.passed && b.passed) return false;
  // Both same pass/fail status — prefer no error
  if (!a.solverResult.error && b.solverResult.error) return true;
  if (a.solverResult.error && !b.solverResult.error) return false;
  // Both same error status — prefer more tool calls (more progress)
  return a.solverResult.toolCalls > b.solverResult.toolCalls;
}

// ---------------------------------------------------------------------------
// Parallel execution with concurrency limit
// ---------------------------------------------------------------------------

/**
 * Run benchmarks with parallel execution and optional retries.
 *
 * @param {object} opts
 * @param {BenchmarkBuilder} opts.builder
 * @param {object} opts.solver
 * @param {string[]} [opts.benchmarkNames]
 * @param {boolean} [opts.runAll=false]
 * @param {number} [opts.levelFilter]
 * @param {string} [opts.tagFilter]
 * @param {number} [opts.concurrency=1] - Max parallel benchmarks
 * @param {number} [opts.retries=1] - Attempts per benchmark
 * @param {Function} [opts.onResult] - Called with (name, result) after each benchmark
 * @param {Function} [opts.onTrace] - Called with trace event objects for observability
 * @returns {Promise<import('./models.js').RunReport>}
 */
export async function runBenchmarks({
  builder, solver, benchmarkNames, runAll = false,
  levelFilter, tagFilter, concurrency = 1, retries = 1,
  onResult, onTrace,
}) {
  let allBenchmarks;
  try {
    allBenchmarks = builder.listBenchmarks();
  } catch {
    builder.clone();
    allBenchmarks = builder.listBenchmarks(true);
  }

  let targets;
  if (benchmarkNames?.length) {
    const available = new Set(allBenchmarks.map((b) => b.dirName));
    targets = benchmarkNames.filter((n) => available.has(n));
  } else if (runAll) {
    targets = allBenchmarks.map((b) => b.dirName);
  } else {
    throw new Error('Must specify benchmarkNames or runAll');
  }

  if (levelFilter != null || tagFilter) {
    const byName = Object.fromEntries(allBenchmarks.map((b) => [b.dirName, b]));
    targets = targets.filter((name) => {
      const b = byName[name];
      if (levelFilter != null && b.level !== levelFilter) return false;
      if (tagFilter && !b.tags.includes(tagFilter)) return false;
      return true;
    });
  }

  const results = [];

  if (concurrency <= 1) {
    // Sequential execution (original behavior)
    for (const name of targets) {
      const startTime = performance.now();
      const result = await runWithRetry(builder, solver, name, { retries });
      const durationMs = performance.now() - startTime;

      results.push(result);
      if (onResult) onResult(name, result);
      if (onTrace) onTrace({
        type: 'benchmark_complete',
        benchmark: name,
        passed: result.passed,
        durationMs: Math.round(durationMs),
        attempts: retries,
        error: result.solverResult.error,
      });
    }
  } else {
    // Parallel execution with concurrency limit
    const inFlight = new Set();
    const queue = [...targets];

    while (queue.length > 0 || inFlight.size > 0) {
      // Fill up to concurrency limit
      while (queue.length > 0 && inFlight.size < concurrency) {
        const name = queue.shift();
        const startTime = performance.now();
        const promise = runWithRetry(builder, solver, name, { retries })
          .then((result) => {
            const durationMs = performance.now() - startTime;
            results.push(result);
            inFlight.delete(promise);
            if (onResult) onResult(name, result);
            if (onTrace) onTrace({
              type: 'benchmark_complete',
              benchmark: name,
              passed: result.passed,
              durationMs: Math.round(durationMs),
              attempts: retries,
              error: result.solverResult.error,
            });
          })
          .catch((err) => {
            results.push({
              config: builder.getBenchmark(name),
              solverResult: new SolverResult({ error: String(err) }),
              passed: false,
              expectedFlag: '',
              actualFlag: '',
            });
            inFlight.delete(promise);
            if (onTrace) onTrace({
              type: 'benchmark_error',
              benchmark: name,
              error: String(err),
            });
          });
        inFlight.add(promise);
      }

      // Wait for at least one to complete
      if (inFlight.size > 0) {
        await Promise.race(inFlight);
      }
    }
  }

  return aggregateResults(results);
}

// ---------------------------------------------------------------------------
// Report serialization
// ---------------------------------------------------------------------------

export function reportToDict(report) {
  const byTag = report.resultsByTag();
  const tagRates = {};
  for (const [tag, tagResults] of Object.entries(byTag)) {
    const p = tagResults.filter(r => r.passed).length;
    tagRates[tag] = {
      total: tagResults.length,
      passed: p,
      rate: tagResults.length ? Math.round((p / tagResults.length) * 1000) / 10 : 0,
    };
  }

  return {
    summary: {
      total: report.total,
      passed: report.passed,
      failed: report.failed,
      skipped: report.skipped,
      pass_rate: Math.round(report.passRate * 10) / 10,
      duration_s: Math.round(report.durationS * 10) / 10,
      total_cost_usd: Math.round(report.totalCostUsd * 10000) / 10000,
      pass_rate_by_level: Object.fromEntries(
        Object.entries(report.passRateByLevel()).map(([k, v]) => [k, Math.round(v * 10) / 10]),
      ),
      pass_rate_by_tag: tagRates,
    },
    results: report.results.map((r) => ({
      benchmark: r.config.dirName,
      name: r.config.name,
      level: r.config.level,
      tags: r.config.tags,
      passed: r.passed,
      expected_flag: r.expectedFlag,
      actual_flag: r.actualFlag,
      target_url: r.targetUrl,
      duration_s: Math.round(r.solverResult.durationS * 1000) / 1000,
      tokens_in: r.solverResult.tokensIn,
      tokens_out: r.solverResult.tokensOut,
      api_cost_usd: r.solverResult.apiCostUsd,
      tool_calls: r.solverResult.toolCalls,
      steps: r.solverResult.steps,
      error: r.solverResult.error,
    })),
  };
}

export { _isBetterResult };
