// Copyright (c) 2026 defconxt. All rights reserved.
// Licensed under AGPL-3.0 — see LICENSE file for details.
// NERF is a trademark of defconxt.

/**
 * Knowledge Solver — MCQ evaluation for security knowledge benchmarks.
 *
 * Evaluates LLM knowledge against multiple-choice question sets:
 * - SecEval: security domain knowledge (2K+ questions)
 * - CyberMetric: cybersecurity metrics and concepts (5K+ questions)
 * - CTIBench: cyber threat intelligence (10K+ questions)
 *
 * Scoring: accuracy (correct/total), with per-category breakdown.
 *
 * @module benchmark/knowledge
 */

import { SolverResult } from './models.js';

// ---------------------------------------------------------------------------
// KnowledgeQuestion
// ---------------------------------------------------------------------------

/**
 * A single MCQ question for knowledge evaluation.
 */
export class KnowledgeQuestion {
  /**
   * @param {object} opts
   * @param {string} opts.id - Question identifier
   * @param {string} opts.question - Question text
   * @param {string[]} opts.choices - Answer choices (e.g. ['A: ...', 'B: ...', ...])
   * @param {string} opts.correctAnswer - Correct answer letter or text
   * @param {string} [opts.category] - Question category
   * @param {string} [opts.difficulty] - Difficulty level
   * @param {string} [opts.source] - Source benchmark (SecEval, CyberMetric, CTIBench)
   */
  constructor(opts = {}) {
    this.id = opts.id ?? '';
    this.question = opts.question ?? '';
    this.choices = opts.choices ?? [];
    this.correctAnswer = opts.correctAnswer ?? '';
    this.category = opts.category ?? '';
    this.difficulty = opts.difficulty ?? '';
    this.source = opts.source ?? '';
  }
}

// ---------------------------------------------------------------------------
// KnowledgeResult
// ---------------------------------------------------------------------------

/**
 * Result of evaluating a set of knowledge questions.
 */
export class KnowledgeResult {
  /**
   * @param {object} opts
   * @param {number} opts.total
   * @param {number} opts.correct
   * @param {number} opts.incorrect
   * @param {number} opts.skipped
   * @param {Record<string, { total: number, correct: number }>} [opts.byCategory]
   * @param {Array<{ id: string, correct: boolean, expected: string, actual: string }>} [opts.details]
   */
  constructor(opts = {}) {
    this.total = opts.total ?? 0;
    this.correct = opts.correct ?? 0;
    this.incorrect = opts.incorrect ?? 0;
    this.skipped = opts.skipped ?? 0;
    this.byCategory = opts.byCategory ?? {};
    this.details = opts.details ?? [];
  }

  /** Accuracy as a percentage. */
  get accuracy() {
    return this.total > 0 ? (this.correct / this.total) * 100 : 0;
  }

  /** Accuracy by category. */
  get categoryAccuracy() {
    const result = {};
    for (const [cat, data] of Object.entries(this.byCategory)) {
      result[cat] = data.total > 0 ? (data.correct / data.total) * 100 : 0;
    }
    return result;
  }
}

// ---------------------------------------------------------------------------
// KnowledgeSolver
// ---------------------------------------------------------------------------

/**
 * Evaluates LLM on MCQ security knowledge questions.
 */
export class KnowledgeSolver {
  /**
   * @param {object} [opts]
   * @param {Function} [opts.agentRunner] - Injectable runner for testing
   */
  constructor(opts = {}) {
    this._agentRunner = opts.agentRunner;
  }

  /**
   * Evaluate a set of questions.
   *
   * @param {KnowledgeQuestion[]} questions
   * @returns {Promise<KnowledgeResult>}
   */
  async evaluate(questions) {
    const details = [];
    const byCategory = {};

    for (const q of questions) {
      const cat = q.category || 'uncategorized';
      if (!byCategory[cat]) byCategory[cat] = { total: 0, correct: 0 };
      byCategory[cat].total++;

      let answer;
      try {
        answer = await this._answerQuestion(q);
      } catch {
        details.push({ id: q.id, correct: false, expected: q.correctAnswer, actual: 'ERROR' });
        continue;
      }

      const isCorrect = this._checkAnswer(answer, q.correctAnswer);
      if (isCorrect) byCategory[cat].correct++;

      details.push({
        id: q.id,
        correct: isCorrect,
        expected: q.correctAnswer,
        actual: answer,
      });
    }

    const correct = details.filter(d => d.correct).length;
    const incorrect = details.filter(d => !d.correct && d.actual !== 'ERROR').length;
    const skipped = details.filter(d => d.actual === 'ERROR').length;

    return new KnowledgeResult({
      total: questions.length,
      correct,
      incorrect,
      skipped,
      byCategory,
      details,
    });
  }

  /**
   * Answer a single question using the LLM.
   *
   * @param {KnowledgeQuestion} question
   * @returns {Promise<string>}
   */
  async _answerQuestion(question) {
    if (this._agentRunner) {
      const result = await this._agentRunner('ARCHITECT', {
        task: 'Answer this security knowledge question',
        user_message: `${question.question}\n\nChoices:\n${question.choices.join('\n')}\n\nRespond with ONLY the correct answer letter (A, B, C, or D).`,
      });
      return (result.outputText || '').trim().charAt(0).toUpperCase();
    }
    throw new Error('No agent runner configured');
  }

  /**
   * Check if the answer matches the correct answer.
   * Handles both letter answers (A, B) and full text matching.
   *
   * @param {string} answer
   * @param {string} correct
   * @returns {boolean}
   */
  _checkAnswer(answer, correct) {
    if (!answer || !correct) return false;
    const a = answer.trim().toUpperCase();
    const c = correct.trim().toUpperCase();
    // Match first letter
    if (a.charAt(0) === c.charAt(0) && /^[A-D]$/.test(a.charAt(0))) return true;
    // Full text match
    return a === c;
  }
}

// ---------------------------------------------------------------------------
// Privacy Solver — PII detection scoring
// ---------------------------------------------------------------------------

/**
 * A single PII detection entry.
 */
export class PIIEntry {
  /**
   * @param {object} opts
   * @param {string} opts.id
   * @param {string} opts.text - Text containing potential PII
   * @param {Array<{ type: string, value: string, start: number, end: number }>} opts.expectedPII
   */
  constructor(opts = {}) {
    this.id = opts.id ?? '';
    this.text = opts.text ?? '';
    this.expectedPII = opts.expectedPII ?? [];
  }
}

/**
 * Result of PII detection evaluation.
 */
export class PIIResult {
  /**
   * @param {object} opts
   * @param {number} opts.total - Total PII entities expected
   * @param {number} opts.truePositives
   * @param {number} opts.falsePositives
   * @param {number} opts.falseNegatives
   */
  constructor(opts = {}) {
    this.total = opts.total ?? 0;
    this.truePositives = opts.truePositives ?? 0;
    this.falsePositives = opts.falsePositives ?? 0;
    this.falseNegatives = opts.falseNegatives ?? 0;
  }

  /** Precision: TP / (TP + FP) */
  get precision() {
    const denom = this.truePositives + this.falsePositives;
    return denom > 0 ? this.truePositives / denom : 0;
  }

  /** Recall: TP / (TP + FN) */
  get recall() {
    const denom = this.truePositives + this.falseNegatives;
    return denom > 0 ? this.truePositives / denom : 0;
  }

  /** F1 score: harmonic mean of precision and recall */
  get f1() {
    const p = this.precision;
    const r = this.recall;
    return (p + r) > 0 ? (2 * p * r) / (p + r) : 0;
  }

  /** F2 score: weighted harmonic mean emphasizing recall */
  get f2() {
    const p = this.precision;
    const r = this.recall;
    return (p + r) > 0 ? (5 * p * r) / (4 * p + r) : 0;
  }
}

// ---------------------------------------------------------------------------
// Accuracy and F-score metrics for scorer.js
// ---------------------------------------------------------------------------

/**
 * Calculate accuracy from correct/total counts.
 * @param {number} correct
 * @param {number} total
 * @returns {number} Accuracy as percentage (0-100)
 */
export function calculateAccuracy(correct, total) {
  return total > 0 ? (correct / total) * 100 : 0;
}

/**
 * Calculate F-beta score.
 * @param {number} precision
 * @param {number} recall
 * @param {number} [beta=1] - Beta weight (1 = F1, 2 = F2)
 * @returns {number}
 */
export function calculateFScore(precision, recall, beta = 1) {
  if (precision + recall === 0) return 0;
  const b2 = beta * beta;
  return ((1 + b2) * precision * recall) / (b2 * precision + recall);
}
