#!/usr/bin/env npx tsx
/**
 * M7: Results Analysis Script
 *
 * Analyzes scan results from real-repo validation and generates
 * summary metrics for documentation.
 *
 * Usage:
 *   npx tsx packages/scanner/src/__tests__/validation/analyze-results.ts
 *   npx tsx packages/scanner/src/__tests__/validation/analyze-results.ts --output docs/RESULTSCOMPARISON.md
 */

import * as fs from 'fs'
import * as path from 'path'
import type { ScanResult, Vulnerability, VulnerabilityCategory, VulnerabilitySeverity } from '../../shared/types'

const RESULTS_DIR = path.join(__dirname, '../../../validation-results')

// M5 AI-era categories we're specifically validating
const M5_CATEGORIES: VulnerabilityCategory[] = [
  'ai_rag_exfiltration',
  'ai_endpoint_unprotected',
  'ai_schema_mismatch',
  'ai_prompt_injection',
  'ai_unsafe_execution',
  'ai_overpermissive_tool',
]

interface FindingSummary {
  category: VulnerabilityCategory
  count: number
  bySeverity: Record<VulnerabilitySeverity, number>
  examples: Array<{
    file: string
    line: number
    title: string
    severity: VulnerabilitySeverity
  }>
}

interface RepoAnalysis {
  repoName: string
  scanDepth: string
  totalFindings: number
  mediumPlusFindings: number
  filesScanned: number
  scanDuration: number
  bySeverity: Record<VulnerabilitySeverity, number>
  byCategory: Record<string, number>
  m5Findings: FindingSummary[]
  topCategories: Array<{ category: string; count: number }>
  libraryCodeFindings: number
  exampleCodeFindings: number
  testCodeFindings: number
}

interface ValidationReport {
  generatedAt: string
  totalRepos: number
  totalScans: number
  analyses: RepoAnalysis[]
  aggregateMetrics: {
    totalFindings: number
    totalMediumPlus: number
    avgFindingsPerRepo: number
    m5DetectorPerformance: Record<string, { total: number; mediumPlus: number }>
  }
}

/**
 * Classify file path as library internal, example, or test code
 */
function classifyFilePath(filePath: string): 'library' | 'example' | 'test' | 'other' {
  const lowerPath = filePath.toLowerCase()

  // Test files
  if (
    lowerPath.includes('__tests__') ||
    lowerPath.includes('/test/') ||
    lowerPath.includes('/tests/') ||
    lowerPath.includes('.test.') ||
    lowerPath.includes('.spec.') ||
    lowerPath.includes('_test.') ||
    lowerPath.includes('/fixtures/')
  ) {
    return 'test'
  }

  // Example/demo files
  if (
    lowerPath.includes('/examples/') ||
    lowerPath.includes('/example/') ||
    lowerPath.includes('/demos/') ||
    lowerPath.includes('/demo/') ||
    lowerPath.includes('/templates/') ||
    lowerPath.includes('/cookbook/')
  ) {
    return 'example'
  }

  // Library internal code
  if (
    lowerPath.includes('/src/') ||
    lowerPath.includes('/lib/') ||
    lowerPath.includes('/libs/') ||
    lowerPath.includes('/packages/')
  ) {
    return 'library'
  }

  return 'other'
}

/**
 * Load scan results from JSON files
 */
function loadResults(): Map<string, ScanResult> {
  const results = new Map<string, ScanResult>()

  if (!fs.existsSync(RESULTS_DIR)) {
    console.error(`Results directory not found: ${RESULTS_DIR}`)
    console.error('Run run-validation.ts first to generate results.')
    process.exit(1)
  }

  const files = fs.readdirSync(RESULTS_DIR).filter(f =>
    f.endsWith('.json') && !f.includes('medium-plus')
  )

  if (files.length === 0) {
    console.error('No result files found in', RESULTS_DIR)
    console.error('Run run-validation.ts first to generate results.')
    process.exit(1)
  }

  for (const file of files) {
    const filePath = path.join(RESULTS_DIR, file)
    const content = fs.readFileSync(filePath, 'utf-8')
    const result = JSON.parse(content) as ScanResult
    const key = file.replace('.json', '')
    results.set(key, result)
  }

  return results
}

/**
 * Analyze a single scan result
 */
function analyzeResult(key: string, result: ScanResult): RepoAnalysis {
  const [repoName, scanDepth] = key.split('-')

  // Severity distribution
  const bySeverity: Record<VulnerabilitySeverity, number> = {
    critical: 0,
    high: 0,
    medium: 0,
    low: 0,
    info: 0,
  }

  // Category distribution
  const byCategory: Record<string, number> = {}

  // Code type distribution
  let libraryCodeFindings = 0
  let exampleCodeFindings = 0
  let testCodeFindings = 0

  // M5 detector findings
  const m5FindingsMap = new Map<VulnerabilityCategory, Vulnerability[]>()
  for (const cat of M5_CATEGORIES) {
    m5FindingsMap.set(cat, [])
  }

  // Process each finding
  for (const vuln of result.vulnerabilities) {
    // Severity
    bySeverity[vuln.severity]++

    // Category
    byCategory[vuln.category] = (byCategory[vuln.category] || 0) + 1

    // Code type
    const codeType = classifyFilePath(vuln.filePath)
    if (codeType === 'library') libraryCodeFindings++
    else if (codeType === 'example') exampleCodeFindings++
    else if (codeType === 'test') testCodeFindings++

    // M5 categories
    if (M5_CATEGORIES.includes(vuln.category as VulnerabilityCategory)) {
      m5FindingsMap.get(vuln.category as VulnerabilityCategory)!.push(vuln)
    }
  }

  // Build M5 findings summary
  const m5Findings: FindingSummary[] = []
  for (const [category, findings] of m5FindingsMap) {
    if (findings.length === 0) continue

    const severityBreakdown: Record<VulnerabilitySeverity, number> = {
      critical: 0,
      high: 0,
      medium: 0,
      low: 0,
      info: 0,
    }

    for (const f of findings) {
      severityBreakdown[f.severity]++
    }

    // Get top 5 examples
    const examples = findings.slice(0, 5).map(f => ({
      file: f.filePath,
      line: f.lineNumber,
      title: f.title,
      severity: f.severity,
    }))

    m5Findings.push({
      category,
      count: findings.length,
      bySeverity: severityBreakdown,
      examples,
    })
  }

  // Sort m5Findings by count
  m5Findings.sort((a, b) => b.count - a.count)

  // Top categories
  const topCategories = Object.entries(byCategory)
    .sort(([, a], [, b]) => b - a)
    .slice(0, 10)
    .map(([category, count]) => ({ category, count }))

  // Medium+ count
  const mediumPlusFindings = bySeverity.critical + bySeverity.high + bySeverity.medium

  return {
    repoName,
    scanDepth,
    totalFindings: result.vulnerabilities.length,
    mediumPlusFindings,
    filesScanned: result.filesScanned,
    scanDuration: result.scanDuration,
    bySeverity,
    byCategory,
    m5Findings,
    topCategories,
    libraryCodeFindings,
    exampleCodeFindings,
    testCodeFindings,
  }
}

/**
 * Generate aggregate metrics across all analyses
 */
function computeAggregateMetrics(analyses: RepoAnalysis[]): ValidationReport['aggregateMetrics'] {
  let totalFindings = 0
  let totalMediumPlus = 0

  const m5Performance: Record<string, { total: number; mediumPlus: number }> = {}
  for (const cat of M5_CATEGORIES) {
    m5Performance[cat] = { total: 0, mediumPlus: 0 }
  }

  for (const analysis of analyses) {
    totalFindings += analysis.totalFindings
    totalMediumPlus += analysis.mediumPlusFindings

    for (const m5 of analysis.m5Findings) {
      m5Performance[m5.category].total += m5.count
      m5Performance[m5.category].mediumPlus +=
        m5.bySeverity.critical + m5.bySeverity.high + m5.bySeverity.medium
    }
  }

  return {
    totalFindings,
    totalMediumPlus,
    avgFindingsPerRepo: analyses.length > 0 ? Math.round(totalFindings / analyses.length) : 0,
    m5DetectorPerformance: m5Performance,
  }
}

/**
 * Generate markdown report
 */
function generateMarkdownReport(report: ValidationReport): string {
  const lines: string[] = []

  lines.push('# M7: Real-Repo Validation Results')
  lines.push('')
  lines.push(`> Generated: ${report.generatedAt}`)
  lines.push('')

  // Executive Summary
  lines.push('## Executive Summary')
  lines.push('')
  lines.push(`- **Repositories scanned:** ${report.totalRepos}`)
  lines.push(`- **Total scans:** ${report.totalScans}`)
  lines.push(`- **Total findings:** ${report.aggregateMetrics.totalFindings}`)
  lines.push(`- **Medium+ findings (to triage):** ${report.aggregateMetrics.totalMediumPlus}`)
  lines.push('')

  // M5 Detector Performance
  lines.push('## M5 AI-Era Detector Performance')
  lines.push('')
  lines.push('| Detector | Total Findings | Medium+ |')
  lines.push('|----------|----------------|---------|')

  for (const [category, stats] of Object.entries(report.aggregateMetrics.m5DetectorPerformance)) {
    if (stats.total > 0) {
      lines.push(`| ${category} | ${stats.total} | ${stats.mediumPlus} |`)
    }
  }

  const emptyM5 = Object.entries(report.aggregateMetrics.m5DetectorPerformance)
    .filter(([, stats]) => stats.total === 0)
    .map(([cat]) => cat)

  if (emptyM5.length > 0) {
    lines.push('')
    lines.push(`**No findings for:** ${emptyM5.join(', ')}`)
  }

  lines.push('')

  // Per-Repo Results
  lines.push('## Per-Repository Results')
  lines.push('')

  // Group analyses by repo
  const byRepo = new Map<string, RepoAnalysis[]>()
  for (const analysis of report.analyses) {
    const existing = byRepo.get(analysis.repoName) || []
    existing.push(analysis)
    byRepo.set(analysis.repoName, existing)
  }

  for (const [repoName, analyses] of byRepo) {
    lines.push(`### ${repoName}`)
    lines.push('')

    for (const analysis of analyses) {
      lines.push(`#### Depth: ${analysis.scanDepth}`)
      lines.push('')
      lines.push(`- Files scanned: ${analysis.filesScanned}`)
      lines.push(`- Total findings: ${analysis.totalFindings}`)
      lines.push(`- Duration: ${analysis.scanDuration}ms`)
      lines.push('')
      lines.push('**Severity Breakdown:**')
      lines.push(`| Critical | High | Medium | Low | Info |`)
      lines.push(`|----------|------|--------|-----|------|`)
      lines.push(`| ${analysis.bySeverity.critical} | ${analysis.bySeverity.high} | ${analysis.bySeverity.medium} | ${analysis.bySeverity.low} | ${analysis.bySeverity.info} |`)
      lines.push('')

      // Code type breakdown
      lines.push('**Finding Distribution by Code Type:**')
      lines.push(`- Library internals: ${analysis.libraryCodeFindings}`)
      lines.push(`- Examples/demos: ${analysis.exampleCodeFindings}`)
      lines.push(`- Test files: ${analysis.testCodeFindings}`)
      lines.push('')

      // Top categories
      if (analysis.topCategories.length > 0) {
        lines.push('**Top Categories:**')
        for (const { category, count } of analysis.topCategories.slice(0, 5)) {
          lines.push(`- ${category}: ${count}`)
        }
        lines.push('')
      }

      // M5 findings with examples
      if (analysis.m5Findings.length > 0) {
        lines.push('**M5 Detector Findings:**')
        for (const m5 of analysis.m5Findings) {
          lines.push(`- **${m5.category}**: ${m5.count} findings`)
          if (m5.examples.length > 0) {
            for (const ex of m5.examples.slice(0, 3)) {
              lines.push(`  - \`${ex.file}:${ex.line}\` (${ex.severity}): ${ex.title}`)
            }
          }
        }
        lines.push('')
      }
    }
  }

  // Triage Checklist
  lines.push('## FP Triage Checklist')
  lines.push('')
  lines.push('For each medium+ finding, classify as:')
  lines.push('- [ ] **True Positive** - Real security issue')
  lines.push('- [ ] **False Positive** - Safe code incorrectly flagged')
  lines.push('- [ ] **Borderline** - Context-dependent, may need tuning')
  lines.push('')
  lines.push('### Triage Template')
  lines.push('')
  lines.push('```markdown')
  lines.push('### Finding: [category]')
  lines.push('**File:** [path]:[line]')
  lines.push('**Severity:** [severity]')
  lines.push('**Classification:** [ ] TP [ ] FP [ ] Borderline')
  lines.push('**Context:** Library internal / Example code / Test file')
  lines.push('**Reasoning:** [Why this is/isn\'t a real issue]')
  lines.push('**Action:** [ ] Keep [ ] Tune detector [ ] Add to known-FP fixtures')
  lines.push('```')
  lines.push('')

  // Next Steps
  lines.push('## Next Steps')
  lines.push('')
  lines.push('1. Review all medium+ findings in this report')
  lines.push('2. Classify each as TP/FP/Borderline')
  lines.push('3. Calculate FP rate: `FP / (TP + FP)` - target: <20%')
  lines.push('4. Add FP patterns to `known-false-positives.test.ts`')
  lines.push('5. Tune detectors if FP rate exceeds 20%')
  lines.push('')

  return lines.join('\n')
}

/**
 * Print console summary
 */
function printConsoleSummary(report: ValidationReport): void {
  console.log('\n' + '='.repeat(60))
  console.log('M7: VALIDATION ANALYSIS SUMMARY')
  console.log('='.repeat(60))

  console.log(`\nGenerated: ${report.generatedAt}`)
  console.log(`Repositories: ${report.totalRepos}`)
  console.log(`Total scans: ${report.totalScans}`)

  console.log('\n--- Aggregate Metrics ---')
  console.log(`Total findings: ${report.aggregateMetrics.totalFindings}`)
  console.log(`Medium+ findings: ${report.aggregateMetrics.totalMediumPlus} (to triage)`)
  console.log(`Avg per repo: ${report.aggregateMetrics.avgFindingsPerRepo}`)

  console.log('\n--- M5 Detector Performance ---')
  console.log('Category                      | Total | Medium+')
  console.log('-'.repeat(50))

  for (const [category, stats] of Object.entries(report.aggregateMetrics.m5DetectorPerformance)) {
    const catPadded = category.padEnd(30)
    console.log(`${catPadded}| ${String(stats.total).padEnd(6)}| ${stats.mediumPlus}`)
  }

  console.log('\n--- Per-Repo Breakdown ---')
  for (const analysis of report.analyses) {
    console.log(`\n${analysis.repoName} (${analysis.scanDepth}):`)
    console.log(`  Files: ${analysis.filesScanned}, Findings: ${analysis.totalFindings}, Medium+: ${analysis.mediumPlusFindings}`)
    console.log(`  Code types: Library=${analysis.libraryCodeFindings}, Example=${analysis.exampleCodeFindings}, Test=${analysis.testCodeFindings}`)
  }
}

/**
 * Main entry point
 */
async function main() {
  const args = process.argv.slice(2)
  let outputPath: string | null = null

  // Parse args
  for (let i = 0; i < args.length; i++) {
    if (args[i] === '--output' && args[i + 1]) {
      outputPath = args[i + 1]
      i++
    } else if (args[i] === '--help' || args[i] === '-h') {
      console.log(`
M7: Results Analysis Script

Usage:
  npx tsx analyze-results.ts [options]

Options:
  --output <path>   Write markdown report to file
  --help, -h        Show this help

Examples:
  npx tsx analyze-results.ts                                    # Print summary
  npx tsx analyze-results.ts --output docs/RESULTSCOMPARISON.md # Generate report
`)
      process.exit(0)
    }
  }

  // Load results
  const results = loadResults()
  console.log(`Loaded ${results.size} result files`)

  // Analyze each result
  const analyses: RepoAnalysis[] = []
  for (const [key, result] of results) {
    const analysis = analyzeResult(key, result)
    analyses.push(analysis)
  }

  // Get unique repo count
  const uniqueRepos = new Set(analyses.map(a => a.repoName))

  // Build report
  const report: ValidationReport = {
    generatedAt: new Date().toISOString(),
    totalRepos: uniqueRepos.size,
    totalScans: analyses.length,
    analyses,
    aggregateMetrics: computeAggregateMetrics(analyses),
  }

  // Print console summary
  printConsoleSummary(report)

  // Generate markdown if output path specified
  if (outputPath) {
    const markdown = generateMarkdownReport(report)
    const fullPath = path.isAbsolute(outputPath)
      ? outputPath
      : path.join(process.cwd(), outputPath)

    // Ensure directory exists
    const dir = path.dirname(fullPath)
    if (!fs.existsSync(dir)) {
      fs.mkdirSync(dir, { recursive: true })
    }

    fs.writeFileSync(fullPath, markdown)
    console.log(`\nMarkdown report saved to: ${fullPath}`)
  } else {
    console.log('\nTip: Use --output docs/RESULTSCOMPARISON.md to generate full report')
  }
}

// Run
main().catch(err => {
  console.error('Analysis failed:', err)
  process.exit(1)
})
