#!/usr/bin/env npx tsx
/**
 * M7 Deep FP Analysis
 *
 * Analyzes the gap between cheap and validated scans to identify
 * exactly which patterns are causing false positives and where
 * to focus tuning efforts.
 */

import * as fs from 'fs'
import * as path from 'path'
import type { ScanResult, Vulnerability } from '../../shared/types'

const RESULTS_DIR = path.join(__dirname, '../../../validation-results')
const OUTPUT_PATH = path.join(__dirname, '../../../docs/FP_DEEP_ANALYSIS.md')

interface RejectedFinding {
  repo: string
  file: string
  line: number
  category: string
  title: string
  severity: string
  lineContent: string
}

interface CategoryAnalysis {
  category: string
  cheapCount: number
  validatedCount: number
  rejectedCount: number
  fpRate: number
  byPathType: Record<string, RejectedFinding[]>
  byTitle: Record<string, number>
  sampleFindings: RejectedFinding[]
}

function classifyPath(filePath: string): string {
  const lower = filePath.toLowerCase()
  if (lower.includes('/examples/') || lower.includes('/example/')) return 'examples'
  if (lower.includes('/__tests__/') || lower.includes('/test/') || lower.includes('.test.') || lower.includes('.spec.')) return 'tests'
  if (lower.includes('/src/') || lower.includes('/lib/') || lower.includes('/libs/')) return 'library'
  return 'other'
}

function loadScanResult(fileName: string): ScanResult | null {
  const filePath = path.join(RESULTS_DIR, fileName)
  if (!fs.existsSync(filePath)) return null
  return JSON.parse(fs.readFileSync(filePath, 'utf-8'))
}

function findRejectedFindings(cheapFile: string, validatedFile: string): RejectedFinding[] {
  const cheap = loadScanResult(cheapFile)
  const validated = loadScanResult(validatedFile)
  if (!cheap || !validated) return []

  // Build set of validated finding keys
  const validatedKeys = new Set(
    validated.vulnerabilities.map(v => `${v.filePath}:${v.lineNumber}:${v.category}`)
  )

  // Find what was rejected (in cheap but not in validated)
  const rejected: RejectedFinding[] = []
  for (const v of cheap.vulnerabilities) {
    if (!['critical', 'high'].includes(v.severity)) continue
    const key = `${v.filePath}:${v.lineNumber}:${v.category}`
    if (!validatedKeys.has(key)) {
      rejected.push({
        repo: cheap.repoName,
        file: v.filePath,
        line: v.lineNumber,
        category: v.category,
        title: v.title,
        severity: v.severity,
        lineContent: v.lineContent?.slice(0, 100) || '',
      })
    }
  }
  return rejected
}

function analyzeCategory(category: string, findings: RejectedFinding[], cheapCount: number, validatedCount: number): CategoryAnalysis {
  const byPathType: Record<string, RejectedFinding[]> = {}
  const byTitle: Record<string, number> = {}

  for (const f of findings) {
    const pathType = classifyPath(f.file)
    if (!byPathType[pathType]) byPathType[pathType] = []
    byPathType[pathType].push(f)

    byTitle[f.title] = (byTitle[f.title] || 0) + 1
  }

  return {
    category,
    cheapCount,
    validatedCount,
    rejectedCount: findings.length,
    fpRate: cheapCount > 0 ? Math.round((findings.length / cheapCount) * 100) : 0,
    byPathType,
    byTitle,
    sampleFindings: findings.slice(0, 10),
  }
}

function generateMarkdown(analyses: CategoryAnalysis[]): string {
  const lines: string[] = []

  lines.push('# M7: False Positive Deep Analysis')
  lines.push('')
  lines.push('> This document analyzes exactly where the scanner is generating false positives')
  lines.push('> to guide targeted improvements to the heuristics.')
  lines.push('')
  lines.push('## Executive Summary')
  lines.push('')
  lines.push('**Problem:** 69% of Critical+High findings in cheap scans are false positives that')
  lines.push('require expensive AI validation to filter. We need to improve the heuristics to')
  lines.push('reduce this noise at the source.')
  lines.push('')
  lines.push('## Category FP Rates (Critical+High only)')
  lines.push('')
  lines.push('| Category | Cheap | Validated | Rejected | FP Rate |')
  lines.push('|----------|-------|-----------|----------|---------|')

  for (const a of analyses.sort((a, b) => b.rejectedCount - a.rejectedCount)) {
    lines.push(`| ${a.category} | ${a.cheapCount} | ${a.validatedCount} | ${a.rejectedCount} | **${a.fpRate}%** |`)
  }

  lines.push('')
  lines.push('## Detailed Analysis by Category')
  lines.push('')

  for (const a of analyses.sort((a, b) => b.rejectedCount - a.rejectedCount)) {
    if (a.rejectedCount === 0) continue

    lines.push(`### ${a.category}`)
    lines.push('')
    lines.push(`- **FP Rate:** ${a.fpRate}% (${a.rejectedCount}/${a.cheapCount} rejected)`)
    lines.push('')

    // Path type breakdown
    lines.push('**Where FPs occur:**')
    lines.push('')
    for (const [pathType, findings] of Object.entries(a.byPathType)) {
      lines.push(`- ${pathType}: ${findings.length} findings (${Math.round((findings.length / a.rejectedCount) * 100)}%)`)
    }
    lines.push('')

    // Title breakdown
    const sortedTitles = Object.entries(a.byTitle).sort((a, b) => b[1] - a[1])
    lines.push('**Common patterns:**')
    lines.push('')
    for (const [title, count] of sortedTitles.slice(0, 5)) {
      lines.push(`- "${title}": ${count} occurrences`)
    }
    lines.push('')

    // Sample findings
    lines.push('**Sample rejected findings:**')
    lines.push('')
    lines.push('```')
    for (const f of a.sampleFindings.slice(0, 5)) {
      lines.push(`${f.file}:${f.line}`)
      lines.push(`  Title: ${f.title}`)
      lines.push(`  Code: ${f.lineContent}`)
      lines.push('')
    }
    lines.push('```')
    lines.push('')

    // Recommendations
    lines.push('**Tuning recommendations:**')
    lines.push('')

    // Category-specific recommendations
    if (a.category === 'ai_endpoint_unprotected') {
      const examplePct = a.byPathType['examples'] ? Math.round((a.byPathType['examples'].length / a.rejectedCount) * 100) : 0
      if (examplePct > 50) {
        lines.push(`- ${examplePct}% of FPs are in /examples/ directories. Add path-based severity downgrade.`)
      }
      lines.push('- Check for global middleware patterns more aggressively')
      lines.push('- Recognize demo/tutorial context from surrounding code')
    } else if (a.category === 'ai_overpermissive_tool') {
      lines.push('- Distinguish between library definitions (intentionally flexible) and app usage')
      lines.push('- Check if tools have sandboxing/restrictions defined elsewhere')
      lines.push('- Look for permission checks in tool implementation')
    } else if (a.category === 'ai_rag_exfiltration') {
      const libPct = a.byPathType['library'] ? Math.round((a.byPathType['library'].length / a.rejectedCount) * 100) : 0
      if (libPct > 50) {
        lines.push(`- ${libPct}% of FPs are in library code. Library base classes are intentionally generic.`)
        lines.push('- Downgrade library code to info severity (consumers add filters)')
      }
      lines.push('- Look for filter parameters in method signatures')
    } else if (a.category === 'ai_unsafe_execution') {
      const examplePct = a.byPathType['examples'] ? Math.round((a.byPathType['examples'].length / a.rejectedCount) * 100) : 0
      lines.push('- Check if path comes from trusted source (config, not user input)')
      if (examplePct > 30) {
        lines.push(`- ${examplePct}% in examples - consider demo context`)
      }
    } else if (a.category === 'hardcoded_secret') {
      lines.push('- These are likely test/fixture data - check file context')
      lines.push('- Look for variable names containing "test", "mock", "example"')
      lines.push('- Check entropy threshold - may be too sensitive')
    }
    lines.push('')
  }

  // Overall recommendations
  lines.push('## Overall Recommendations')
  lines.push('')
  lines.push('### Quick Wins (High Impact, Low Effort)')
  lines.push('')
  lines.push('1. **Path-based severity adjustment:** Downgrade findings in `/examples/` directories to info')
  lines.push('2. **Library code handling:** Flag library base classes as "intentionally generic" with lower severity')
  lines.push('3. **Test file handling:** Already done, but verify it covers all patterns')
  lines.push('')
  lines.push('### Medium-Term Improvements')
  lines.push('')
  lines.push('1. **Better context detection:** Look at surrounding code for security indicators')
  lines.push('2. **Cross-file analysis:** Check if protection exists in middleware/imports')
  lines.push('3. **Comment analysis:** Look for "// example", "// demo", "// for testing" patterns')
  lines.push('')
  lines.push('### Cost Reduction Strategy')
  lines.push('')
  lines.push('If we can reduce the FP rate from 69% to 30% through heuristic improvements:')
  lines.push('- AI validation calls would drop by ~57%')
  lines.push('- $3 scan cost would become ~$1.30')
  lines.push('- Better user experience (less noise to review)')
  lines.push('')

  return lines.join('\n')
}

async function main() {
  console.log('Loading scan results...')

  // Find rejected findings from all repos
  const allRejected: RejectedFinding[] = [
    ...findRejectedFindings('ai-cheap.json', 'ai-validated.json'),
    ...findRejectedFindings('langchainjs-cheap.json', 'langchainjs-validated.json'),
    ...findRejectedFindings('anthropic-cookbook-cheap.json', 'anthropic-cookbook-validated.json'),
    ...findRejectedFindings('openai-cookbook-cheap.json', 'openai-cookbook-validated.json'),
  ]

  console.log(`Found ${allRejected.length} rejected findings (FPs)`)

  // Load cheap scan totals by category
  const cheapTotals: Record<string, number> = {}
  const validatedTotals: Record<string, number> = {}

  const cheapFiles = ['ai-cheap.json', 'langchainjs-cheap.json', 'anthropic-cookbook-cheap.json', 'openai-cookbook-cheap.json']
  const validatedFiles = ['ai-validated.json', 'langchainjs-validated.json', 'anthropic-cookbook-validated.json', 'openai-cookbook-validated.json']

  for (const f of cheapFiles) {
    const result = loadScanResult(f)
    if (!result) continue
    for (const v of result.vulnerabilities) {
      if (!['critical', 'high'].includes(v.severity)) continue
      cheapTotals[v.category] = (cheapTotals[v.category] || 0) + 1
    }
  }

  for (const f of validatedFiles) {
    const result = loadScanResult(f)
    if (!result) continue
    for (const v of result.vulnerabilities) {
      if (!['critical', 'high'].includes(v.severity)) continue
      validatedTotals[v.category] = (validatedTotals[v.category] || 0) + 1
    }
  }

  // Group rejected by category
  const byCategory: Record<string, RejectedFinding[]> = {}
  for (const f of allRejected) {
    if (!byCategory[f.category]) byCategory[f.category] = []
    byCategory[f.category].push(f)
  }

  // Analyze each category
  const analyses: CategoryAnalysis[] = []
  for (const [category, findings] of Object.entries(byCategory)) {
    analyses.push(analyzeCategory(
      category,
      findings,
      cheapTotals[category] || 0,
      validatedTotals[category] || 0
    ))
  }

  // Also add categories with 0% FP rate
  for (const [category, count] of Object.entries(cheapTotals)) {
    if (!byCategory[category]) {
      analyses.push({
        category,
        cheapCount: count,
        validatedCount: validatedTotals[category] || 0,
        rejectedCount: 0,
        fpRate: 0,
        byPathType: {},
        byTitle: {},
        sampleFindings: [],
      })
    }
  }

  // Generate report
  const markdown = generateMarkdown(analyses)

  // Ensure output directory exists
  const outputDir = path.dirname(OUTPUT_PATH)
  if (!fs.existsSync(outputDir)) {
    fs.mkdirSync(outputDir, { recursive: true })
  }

  fs.writeFileSync(OUTPUT_PATH, markdown)
  console.log(`\nReport saved to: ${OUTPUT_PATH}`)

  // Print summary
  console.log('\n=== SUMMARY ===')
  console.log('Categories with highest FP rates:')
  for (const a of analyses.sort((a, b) => b.fpRate - a.fpRate).slice(0, 5)) {
    if (a.cheapCount === 0) continue
    console.log(`  ${a.category}: ${a.fpRate}% FP rate (${a.rejectedCount}/${a.cheapCount})`)
  }
}

main().catch(console.error)
