// Copyright (c) 2026 defconxt. All rights reserved.
// Licensed under AGPL-3.0 — see LICENSE file for details.
// NERF is a trademark of defconxt.

/**
 * Knowledge indexer — reads knowledge/ docs and skills/ SKILL.md files,
 * chunks by ## section headers, extracts metadata (domain, mode_tags,
 * vuln_class, mitre_attack), and writes to the KnowledgeStore FTS5 database.
 *
 * Usage:
 *   node cli/lib/knowledge/indexer.js --rebuild
 */

import { KnowledgeStore, DEFAULT_DB_PATH } from './store.js';
import { parse as parseYaml } from 'yaml';
import { readFileSync, readdirSync, statSync, existsSync, unlinkSync } from 'node:fs';
import { join, dirname, basename, relative } from 'node:path';
import { createHash } from 'node:crypto';
import { fileURLToPath } from 'node:url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

// ---------------------------------------------------------------------------
// Path resolution — findContentRoot()
// ---------------------------------------------------------------------------

/**
 * Walk up from __dirname looking for a directory that contains both
 * `skills/` and either `cli/` or `knowledge/`. Works in git clones and
 * npm installs.
 */
function findContentRoot() {
  let dir = dirname(dirname(dirname(__dirname))); // start at cli/lib/knowledge -> repo root
  for (let i = 0; i < 10; i++) {
    const hasSkills = existsSync(join(dir, 'skills')) || existsSync(join(dir, 'src', 'resources', 'skills'));
    if (hasSkills && (existsSync(join(dir, 'cli')) || existsSync(join(dir, 'knowledge')))) {
      return dir;
    }
    const parent = dirname(dir);
    if (parent === dir) break;
    dir = parent;
  }
  // Fallback: assume three levels up from __dirname is the root
  return dirname(dirname(dirname(__dirname)));
}

// ---------------------------------------------------------------------------
// YAML frontmatter parser
// ---------------------------------------------------------------------------

/**
 * Extract YAML frontmatter between --- delimiters.
 * Handles leading HTML comments (<!-- ... -->).
 * @returns {{ frontmatter: object, body: string }}
 */
function parseYamlFrontmatter(content) {
  // Strip leading HTML comments
  let stripped = content.replace(/^(\s*<!--[\s\S]*?-->\s*)+/, '');

  // Match --- delimited frontmatter
  const match = stripped.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/);
  if (!match) {
    return { frontmatter: {}, body: content };
  }

  try {
    const frontmatter = parseYaml(match[1]) || {};
    return { frontmatter, body: match[2] };
  } catch {
    // Malformed YAML — skip frontmatter, use full content as body
    return { frontmatter: {}, body: content };
  }
}

// ---------------------------------------------------------------------------
// Markdown chunker
// ---------------------------------------------------------------------------

/**
 * Split markdown body on ## headers. ### subsections stay within the parent
 * chunk UNLESS the parent is very large (> 3000 chars), in which case ###
 * subsections become their own chunks to improve FTS5 ranking precision.
 * Returns array of { heading, content }.
 */
function chunkMarkdown(body) {
  const chunks = [];
  // Split on lines that start with ## (but not ### or more)
  const parts = body.split(/^(?=## (?!#))/m);

  for (const part of parts) {
    const trimmed = part.trim();
    if (!trimmed) continue;

    const headingMatch = trimmed.match(/^## (.+)/);
    if (headingMatch) {
      // If the chunk is large, sub-split on ### to improve search precision
      if (trimmed.length > 3000) {
        const subParts = trimmed.split(/^(?=### )/m);
        for (const sub of subParts) {
          const subTrimmed = sub.trim();
          if (!subTrimmed || subTrimmed.length < 20) continue;
          const subHeading = subTrimmed.match(/^###? (.+)/);
          chunks.push({
            heading: subHeading ? subHeading[1].trim() : headingMatch[1].trim(),
            content: subTrimmed,
          });
        }
      } else {
        chunks.push({
          heading: headingMatch[1].trim(),
          content: trimmed,
        });
      }
    } else if (chunks.length === 0) {
      // Content before the first ## heading — treat as intro chunk
      chunks.push({
        heading: '(intro)',
        content: trimmed,
      });
    }
  }

  // Filter out tiny chunks (< 20 chars of content) to avoid noise
  return chunks.filter(c => c.content.length >= 20);
}

// ---------------------------------------------------------------------------
// Metadata extraction helpers
// ---------------------------------------------------------------------------

/** Filename prefix → domain mapping for knowledge docs */
const FILENAME_DOMAIN_MAP = {
  'websec': 'web-security',
  'api-exploitation': 'api-security',
  'active-directory': 'active-directory',
  'windows-ad': 'active-directory',
  'windows-internals': 'windows-security',
  'windows-eventlog': 'windows-security',
  'binary-exploitation': 'binary-exploitation',
  'linux-exploitation': 'linux-security',
  'malware': 'malware-analysis',
  'dfir': 'digital-forensics',
  'forensics': 'digital-forensics',
  'network-forensics': 'network-forensics',
  'email-forensics': 'digital-forensics',
  'timeline-analysis': 'digital-forensics',
  'incident-playbooks': 'incident-response',
  'secops-runbooks': 'incident-response',
  'threat-hunting': 'threat-hunting',
  'threat-intel': 'threat-intelligence',
  'threat-modeling': 'threat-modeling',
  'sigma-detection': 'detection-engineering',
  'evasion': 'evasion-techniques',
  'siem-soc': 'soc-operations',
  'defensive': 'defensive-security',
  'edr-av': 'endpoint-security',
  'offensive': 'offensive-security',
  'redteam': 'red-team',
  'c2-postexploit': 'post-exploitation',
  'shells-arsenal': 'post-exploitation',
  'pentest': 'penetration-testing',
  'bugbounty': 'bug-bounty',
  'recon-osint': 'osint',
  'osint': 'osint',
  'social-engineering': 'social-engineering',
  'email-phishing': 'social-engineering',
  'password-credential': 'credential-attacks',
  'identity-auth': 'authentication',
  'cloud-attacks': 'cloud-security',
  'cloud-infra': 'cloud-security',
  'aws-security': 'cloud-security',
  'azure-security': 'cloud-security',
  'gcp-security': 'cloud-security',
  'container-k8s': 'container-security',
  'kubernetes': 'container-security',
  'network-attacks': 'network-security',
  'network-protocol': 'network-security',
  'network-segmentation': 'network-security',
  'dns-email-infra': 'infrastructure-security',
  'wireless-physical-iot': 'iot-security',
  'ics-scada': 'ics-security',
  'mobile-security': 'mobile-security',
  'blockchain-web3': 'blockchain-security',
  'crypto-pki-tls': 'cryptography',
  'privacy': 'privacy',
  'data-protection': 'data-protection',
  'compliance': 'compliance',
  'grc-risk': 'compliance',
  'devsecops': 'devsecops',
  'dev-security': 'devsecops',
  'secure-coding': 'secure-development',
  'secure-infra': 'security-architecture',
  'security-architecture': 'security-architecture',
  'security-automation': 'security-automation',
  'hardening': 'hardening',
  'supply-chain': 'supply-chain-security',
  'logging-monitoring': 'logging-monitoring',
  'ai-defense': 'ai-security',
  'emerging-threats': 'emerging-threats',
  'encoding-manipulation': 'web-security',
  'exfil-tunneling': 'exfiltration',
  'insider-threat': 'insider-threat',
  'purple-team': 'purple-team',
  'vuln-research': 'vulnerability-research',
  'mitre-attack': 'threat-intelligence',
  'ctf': 'ctf',
  'attack-chains': 'offensive-security',
  'breach-case': 'incident-response',
  'security-cert': 'security-training',
  'security-leader': 'security-management',
  'security-mastery': 'security-training',
  'security-metrics': 'security-management',
  'security-scenarios': 'security-training',
  'startup-security': 'security-architecture',
  'powershell': 'windows-security',
  'supplementary': 'general-security',
  'curated-resources': 'general-security',
  'index': 'general-security',
};

/** Domain → mode tags mapping */
const MODE_TAG_MAP = {
  // RED mode domains
  'web-security': ['red'],
  'api-security': ['red'],
  'binary-exploitation': ['red'],
  'linux-security': ['red'],
  'offensive-security': ['red'],
  'red-team': ['red'],
  'post-exploitation': ['red'],
  'penetration-testing': ['red'],
  'bug-bounty': ['red'],
  'credential-attacks': ['red'],
  'evasion-techniques': ['red'],
  'exfiltration': ['red'],
  'social-engineering': ['red'],
  'vulnerability-research': ['red'],

  // BLUE mode domains
  'defensive-security': ['blue'],
  'detection-engineering': ['blue'],
  'digital-forensics': ['blue'],
  'network-forensics': ['blue'],
  'incident-response': ['blue'],
  'soc-operations': ['blue'],
  'threat-hunting': ['blue'],
  'endpoint-security': ['blue'],
  'logging-monitoring': ['blue'],
  'hardening': ['blue'],

  // Both RED and BLUE
  'active-directory': ['red', 'blue'],
  'windows-security': ['red', 'blue'],
  'cloud-security': ['red', 'blue'],
  'container-security': ['red', 'blue'],
  'network-security': ['red', 'blue'],
  'authentication': ['red', 'blue'],
  'infrastructure-security': ['red', 'blue'],
  'threat-intelligence': ['red', 'blue'],
  'threat-modeling': ['red', 'blue'],
  'purple-team': ['red', 'blue'],
  'malware-analysis': ['red', 'blue'],
  'osint': ['red', 'recon'],
  'ctf': ['red'],

  // PRIVACY mode domains
  'privacy': ['privacy'],
  'data-protection': ['privacy'],
  'compliance': ['privacy', 'blue'],

  // ARCHITECT mode domains
  'security-architecture': ['architect'],
  'devsecops': ['architect', 'blue'],
  'secure-development': ['architect'],
  'supply-chain-security': ['architect', 'blue'],
  'security-automation': ['architect', 'blue'],

  // Other
  'ai-security': ['red', 'blue'],
  'blockchain-security': ['red'],
  'cryptography': ['red', 'blue'],
  'ics-security': ['red', 'blue'],
  'iot-security': ['red', 'blue'],
  'mobile-security': ['red', 'blue'],
  'emerging-threats': ['red', 'blue'],
  'insider-threat': ['blue'],
  'security-training': ['red', 'blue'],
  'security-management': ['blue', 'architect'],
  'general-security': ['red', 'blue'],
};

/**
 * Infer domain from file path and frontmatter.
 */
function inferDomain(sourceFile, frontmatter) {
  // Prefer frontmatter domain + subdomain
  if (frontmatter.subdomain) return frontmatter.subdomain;
  if (frontmatter.domain && frontmatter.domain !== 'cybersecurity') return frontmatter.domain;

  // For knowledge docs, infer from filename prefix
  const fname = basename(sourceFile).replace(/\.md$/, '');

  // Try longest prefix match first
  const prefixes = Object.keys(FILENAME_DOMAIN_MAP).sort((a, b) => b.length - a.length);
  for (const prefix of prefixes) {
    if (fname.startsWith(prefix)) {
      return FILENAME_DOMAIN_MAP[prefix];
    }
  }

  // For skills, use the directory name
  const parts = sourceFile.split(/[/\\]/);
  const skillsIdx = parts.indexOf('skills');
  if (skillsIdx >= 0 && skillsIdx + 1 < parts.length) {
    return parts[skillsIdx + 1];
  }

  return '';
}

/**
 * Map domain to applicable NERF modes.
 */
function inferModeTags(domain) {
  if (!domain) return ['red', 'blue'];
  return MODE_TAG_MAP[domain] || ['red', 'blue'];
}

/** Vuln class patterns — regex → canonical name */
const VULN_CLASS_PATTERNS = [
  [/sql\s*injection/i, 'sqli'],
  [/cross[- ]?site\s*scripting|(?<!\w)xss(?!\w)/i, 'xss'],
  [/server[- ]?side\s*template\s*injection|(?<!\w)ssti(?!\w)/i, 'ssti'],
  [/(?<!\w)ssrf(?!\w)|server[- ]?side\s*request\s*forgery/i, 'ssrf'],
  [/file\s*inclusion|(?<!\w)lfi(?!\w)|(?<!\w)rfi(?!\w)/i, 'lfi'],
  [/deserialization/i, 'deserialization'],
  [/(?<!\w)xxe(?!\w)|xml\s*external\s*entity/i, 'xxe'],
  [/command\s*injection|(?<!\w)rce(?!\w)|remote\s*code\s*execution/i, 'rce'],
  [/(?<!\w)csrf(?!\w)|cross[- ]?site\s*request\s*forgery/i, 'csrf'],
  [/path\s*traversal|directory\s*traversal/i, 'path-traversal'],
  [/(?<!\w)idor(?!\w)|insecure\s*direct\s*object/i, 'idor'],
  [/prototype\s*pollution/i, 'prototype-pollution'],
  [/(?<!\w)jwt(?!\w).*(?:attack|vuln|bypass|forg)/i, 'jwt'],
  [/request\s*smuggling/i, 'request-smuggling'],
  [/cache\s*poisoning/i, 'cache-poisoning'],
  [/(?<!\w)csp(?!\w).*bypass/i, 'csp-bypass'],
  [/(?<!\w)cors(?!\w)/i, 'cors'],
  [/open\s*redirect/i, 'open-redirect'],
];

/**
 * Infer vulnerability class from heading and content.
 */
function inferVulnClass(heading, content) {
  const text = heading + ' ' + content.slice(0, 500);
  for (const [pattern, cls] of VULN_CLASS_PATTERNS) {
    if (pattern.test(text)) return cls;
  }
  return '';
}

/**
 * Extract MITRE ATT&CK technique IDs from frontmatter and content.
 */
function extractMitreIds(content, frontmatter) {
  const ids = new Set();

  // From frontmatter metadata.mitre-attack array
  const fmIds = frontmatter?.metadata?.['mitre-attack'] || frontmatter?.['mitre-attack'] || [];
  for (const id of fmIds) {
    if (typeof id === 'string' && /^T\d{4}/.test(id)) ids.add(id);
  }

  // From content: T1234 or T1234.001
  const matches = content.match(/T\d{4}(?:\.\d{3})?/g);
  if (matches) {
    for (const m of matches) ids.add(m);
  }

  return [...ids];
}

/**
 * Generate a deterministic chunk ID from source file and heading.
 */
function makeChunkId(sourceFile, heading) {
  return createHash('sha256')
    .update(sourceFile + '::' + heading)
    .digest('hex')
    .slice(0, 16);
}

// ---------------------------------------------------------------------------
// File discovery
// ---------------------------------------------------------------------------

/**
 * Recursively walk a directory, calling callback for each file.
 */
function walkDir(dir, callback) {
  let entries;
  try {
    entries = readdirSync(dir, { withFileTypes: true });
  } catch {
    return;
  }
  for (const entry of entries) {
    const fullPath = join(dir, entry.name);
    if (entry.isDirectory()) {
      walkDir(fullPath, callback);
    } else if (entry.isFile()) {
      callback(fullPath);
    }
  }
}

/**
 * Discover all indexable files from knowledge/ and skills/ directories.
 * Excludes knowledge/distilled/ and 00-MASTER-INDEX.md.
 */
function discoverFiles(root) {
  const files = [];
  const knowledgeDir = join(root, 'knowledge');
  const skillsDir = existsSync(join(root, 'skills'))
    ? join(root, 'skills')
    : join(root, 'src', 'resources', 'skills');

  // Knowledge docs — top-level .md files only, exclude distilled/ and index files
  if (existsSync(knowledgeDir)) {
    try {
      for (const entry of readdirSync(knowledgeDir, { withFileTypes: true })) {
        if (!entry.isFile() || !entry.name.endsWith('.md')) continue;
        if (entry.name === '00-MASTER-INDEX.md') continue;
        // Check if the file is in the distilled/ subdirectory
        const fullPath = join(knowledgeDir, entry.name);
        files.push({ path: fullPath, type: 'knowledge' });
      }
    } catch {
      process.stderr.write('[indexer] WARN: cannot read knowledge/ directory\n');
    }
  }

  // Exclude distilled subdirectory explicitly
  // (We only read top-level knowledge/*.md above, so distilled/ is already excluded)

  // Skills — recursive find for SKILL.md files
  if (existsSync(skillsDir)) {
    walkDir(skillsDir, (filePath) => {
      if (basename(filePath) === 'SKILL.md') {
        files.push({ path: filePath, type: 'skill' });
      }
    });
  }

  return files;
}

// ---------------------------------------------------------------------------
// buildIndex — main orchestration
// ---------------------------------------------------------------------------

/**
 * Build the full knowledge index.
 *
 * @param {object} [opts]
 * @param {boolean} [opts.rebuild] — delete existing DB before building
 * @returns {{ files: number, chunks: number, skipped: number }}
 */
export function buildIndex(opts = {}) {
  const startTime = Date.now();
  const root = findContentRoot();

  // --rebuild: delete existing DB file
  if (opts.rebuild && existsSync(DEFAULT_DB_PATH)) {
    try {
      unlinkSync(DEFAULT_DB_PATH);
      // Also remove WAL and SHM files
      if (existsSync(DEFAULT_DB_PATH + '-wal')) unlinkSync(DEFAULT_DB_PATH + '-wal');
      if (existsSync(DEFAULT_DB_PATH + '-shm')) unlinkSync(DEFAULT_DB_PATH + '-shm');
    } catch (err) {
      process.stderr.write(`[indexer] WARN: failed to delete old DB: ${err.message}\n`);
    }
  }

  const store = new KnowledgeStore();
  const files = discoverFiles(root);
  process.stderr.write(`[indexer] Discovered ${files.length} files (root: ${root})\n`);

  let totalChunks = 0;
  let skippedFiles = 0;
  let processedFiles = 0;

  // Wrap all inserts in a single transaction for speed
  const insertAll = store.db.transaction(() => {
    for (const { path: filePath, type: fileType } of files) {
      let content;
      try {
        content = readFileSync(filePath, 'utf8');
      } catch (err) {
        process.stderr.write(`[indexer] SKIP: ${filePath} — ${err.message}\n`);
        skippedFiles++;
        continue;
      }

      const relPath = relative(root, filePath);
      const { frontmatter, body } = parseYamlFrontmatter(content);
      const chunks = chunkMarkdown(body);

      if (chunks.length === 0) {
        // File had no ## headers — treat the whole body as one chunk
        if (body.trim().length >= 20) {
          chunks.push({
            heading: basename(filePath, '.md'),
            content: body.trim(),
          });
        }
      }

      const domain = inferDomain(relPath, frontmatter);
      const modeTags = inferModeTags(domain);
      const mitreIds = extractMitreIds(content, frontmatter);

      for (let ci = 0; ci < chunks.length; ci++) {
        const chunk = chunks[ci];
        const chunkId = makeChunkId(relPath, chunk.heading + '#' + ci);
        const vulnClass = inferVulnClass(chunk.heading, chunk.content);
        const chunkMitre = extractMitreIds(chunk.content, {});
        // Merge file-level and chunk-level MITRE IDs
        const allMitre = [...new Set([...mitreIds, ...chunkMitre])];

        store.insertChunk({
          chunk_id: chunkId,
          source_file: relPath,
          section_heading: chunk.heading,
          content: chunk.content,
          domain,
          subdomain: frontmatter.subdomain || '',
          mode_tags: modeTags,
          vuln_class: vulnClass,
          mitre_attack: allMitre,
          file_type: fileType,
        });
        totalChunks++;
      }

      processedFiles++;
    }
  });

  insertAll();
  store.close();

  const elapsed = ((Date.now() - startTime) / 1000).toFixed(2);
  process.stderr.write(
    `[indexer] Done: ${processedFiles} files, ${totalChunks} chunks, ${skippedFiles} skipped in ${elapsed}s\n`
  );

  return { files: processedFiles, chunks: totalChunks, skipped: skippedFiles };
}

// ---------------------------------------------------------------------------
// CLI entry
// ---------------------------------------------------------------------------

// Detect if run directly (not imported)
const isMain = process.argv[1] && (
  process.argv[1].endsWith('indexer.js') ||
  process.argv[1].endsWith('knowledge/indexer.js')
);

if (isMain) {
  const rebuild = process.argv.includes('--rebuild');
  buildIndex({ rebuild });
}
