/**
 * Evaluation Harness
 *
 * Runs scanners against test fixtures and collects results
 * for precision/recall calculation.
 *
 * @module eval/harness
 */
import { mkdir, writeFile, rm } from "fs/promises";
import { join } from "path";
import { tmpdir } from "os";
import { runAllScanners } from "../scanners/index.js";
import { ALL_FIXTURES } from "./fixtures.js";
/**
 * Run evaluation on a single fixture
 */
export async function evaluateFixture(fixture, config) {
    const startTime = Date.now();
    // Create temporary directory for fixture
    const tempDir = join(tmpdir(), `vaspera-eval-${fixture.id}-${Date.now()}`);
    try {
        await mkdir(tempDir, { recursive: true });
        // Write fixture files
        for (const file of fixture.files) {
            const filePath = join(tempDir, file.path);
            const fileDir = join(tempDir, file.path.split("/").slice(0, -1).join("/"));
            if (fileDir !== tempDir) {
                await mkdir(fileDir, { recursive: true });
            }
            await writeFile(filePath, file.content, "utf-8");
        }
        // Create minimal package.json and tsconfig if needed
        const hasTs = fixture.files.some((f) => f.language === "typescript" || f.language === "tsx");
        if (hasTs) {
            await writeFile(join(tempDir, "tsconfig.json"), JSON.stringify({
                compilerOptions: {
                    target: "ES2020",
                    module: "ESNext",
                    strict: true,
                    esModuleInterop: true,
                    skipLibCheck: true,
                },
                include: ["**/*.ts", "**/*.tsx"],
            }), "utf-8");
        }
        await writeFile(join(tempDir, "package.json"), JSON.stringify({
            name: `fixture-${fixture.id}`,
            version: "1.0.0",
            type: "module",
        }), "utf-8");
        // Build scanner options
        const scannerOptions = {
            semgrep: config.scanners.includes("semgrep"),
            dependencies: config.scanners.includes("dependencies"),
            secrets: config.scanners.includes("secrets") || config.scanners.includes("gitleaks"),
            typescript: config.scanners.includes("typescript") || config.scanners.includes("tsc"),
            timeout: config.timeout,
        };
        // Run scanners
        const scanResult = await runAllScanners(tempDir, scannerOptions);
        // Convert findings to ActualFinding format
        const actualFindings = [];
        for (const scanner of scanResult.scanners) {
            for (const finding of scanner.findings) {
                actualFindings.push({
                    scanner: finding.scanner,
                    ruleId: finding.ruleId,
                    file: finding.file,
                    line: finding.line,
                    severity: finding.severity,
                    message: finding.message,
                    confidence: finding.confidence,
                });
            }
        }
        // Match findings against expected
        const { truePositives, falsePositives, falseNegatives } = matchFindings(actualFindings, fixture.expectedFindings);
        return {
            fixtureId: fixture.id,
            success: true,
            actualFindings,
            expectedFindings: fixture.expectedFindings,
            truePositives,
            falsePositives,
            falseNegatives,
            duration: Date.now() - startTime,
        };
    }
    catch (error) {
        return {
            fixtureId: fixture.id,
            success: false,
            error: error instanceof Error ? error.message : String(error),
            actualFindings: [],
            expectedFindings: fixture.expectedFindings,
            truePositives: [],
            falsePositives: [],
            falseNegatives: fixture.expectedFindings,
            duration: Date.now() - startTime,
        };
    }
    finally {
        // Cleanup
        try {
            await rm(tempDir, { recursive: true, force: true });
        }
        catch {
            // Ignore cleanup errors
        }
    }
}
/**
 * Match actual findings against expected findings
 */
export function matchFindings(actual, expected) {
    const truePositives = [];
    const matchedExpectedIndices = new Set();
    const matchedActualIndices = new Set();
    // For each expected finding, try to find a matching actual finding
    for (let ei = 0; ei < expected.length; ei++) {
        const exp = expected[ei];
        // Only look for matches for vulnerable findings
        if (!exp.isVulnerable)
            continue;
        for (let ai = 0; ai < actual.length; ai++) {
            if (matchedActualIndices.has(ai))
                continue;
            const act = actual[ai];
            // Check if this actual finding matches the expected one
            if (findingsMatch(act, exp)) {
                truePositives.push({ expected: exp, actual: act });
                matchedExpectedIndices.add(ei);
                matchedActualIndices.add(ai);
                break;
            }
        }
    }
    // False positives: actual findings that didn't match any expected
    const falsePositives = actual.filter((_, i) => !matchedActualIndices.has(i));
    // False negatives: expected findings that weren't matched
    const falseNegatives = expected.filter((exp, i) => exp.isVulnerable && !matchedExpectedIndices.has(i));
    return { truePositives, falsePositives, falseNegatives };
}
/**
 * Check if an actual finding matches an expected finding
 */
function findingsMatch(actual, expected) {
    // File must match (case-insensitive, handle relative paths)
    const actualFile = actual.file.toLowerCase().replace(/^\.\//, "");
    const expectedFile = expected.file.toLowerCase().replace(/^\.\//, "");
    if (!actualFile.endsWith(expectedFile) && !expectedFile.endsWith(actualFile)) {
        return false;
    }
    // Line must be within 5 lines (scanners may report slightly different lines)
    const lineDiff = Math.abs(actual.line - expected.line);
    if (lineDiff > 5) {
        return false;
    }
    // Category should roughly match
    // This is fuzzy because scanners use different categorization
    const categoryMatch = actual.message.toLowerCase().includes(expected.category.replace("-", " ")) ||
        actual.message.toLowerCase().includes(expected.category.replace("-", "")) ||
        actual.ruleId.toLowerCase().includes(expected.category.replace("-", ""));
    // If category doesn't match, check CWE
    if (!categoryMatch && expected.cweId) {
        const cweMatch = actual.ruleId.includes(expected.cweId) ||
            actual.message.includes(expected.cweId);
        if (!cweMatch) {
            // Allow match if severity is similar and within line range
            // This handles cases where the scanner uses different naming
            return lineDiff <= 2;
        }
    }
    return true;
}
/**
 * Run evaluation on all fixtures
 */
export async function runEvaluation(config = {}) {
    const fullConfig = {
        scanners: config.scanners || ["semgrep", "gitleaks", "tsc"],
        stabilityRuns: config.stabilityRuns || 1,
        timeout: config.timeout || 60000,
        parallel: config.parallel ?? true,
        includeFixtures: config.includeFixtures,
        excludeFixtures: config.excludeFixtures,
    };
    // Filter fixtures
    let fixtures = [...ALL_FIXTURES];
    if (fullConfig.includeFixtures?.length) {
        fixtures = fixtures.filter((f) => fullConfig.includeFixtures.some((pattern) => f.id.includes(pattern) || f.category.includes(pattern)));
    }
    if (fullConfig.excludeFixtures?.length) {
        fixtures = fixtures.filter((f) => !fullConfig.excludeFixtures.some((pattern) => f.id.includes(pattern) || f.category.includes(pattern)));
    }
    // Run evaluations
    if (fullConfig.parallel) {
        const results = await Promise.all(fixtures.map((fixture) => evaluateFixture(fixture, fullConfig)));
        return results;
    }
    else {
        const results = [];
        for (const fixture of fixtures) {
            const result = await evaluateFixture(fixture, fullConfig);
            results.push(result);
        }
        return results;
    }
}
/**
 * Run evaluation multiple times for stability testing
 */
export async function runStabilityTest(config = {}, runs = 5) {
    const allResults = [];
    for (let i = 0; i < runs; i++) {
        const results = await runEvaluation(config);
        allResults.push(results);
    }
    return allResults;
}
//# sourceMappingURL=harness.js.map