/**
 * Tests for multi-model consensus calculation
 */
import { describe, it, expect } from "vitest";
import { calculateFindingSimilarity, calculateStringSimilarity, matchFindings, findDisagreements, calculateFleissKappa, mergeFindings, calculateMultiModelConsensus, generateConsensusSummary, } from "./consensus.js";
describe("Multi-Model Consensus", () => {
    // Helper to create mock findings
    function createFinding(overrides = {}) {
        return {
            id: `finding-${Math.random().toString(36).slice(2, 8)}`,
            description: overrides.description || "A test finding description",
            severity: "medium",
            category: "security",
            evidence: "Test evidence",
            confidence: 80,
            verifications: [],
            created_at: new Date().toISOString(),
            ...overrides,
        };
    }
    // Helper to create mock model results
    function createModelResult(model, findings, success = true) {
        const providers = {
            "claude-3.5-sonnet": "anthropic",
            "gpt-4-turbo": "openai",
            "gemini-1.5-pro": "google",
        };
        return {
            model,
            provider: providers[model],
            agent: "security",
            findings,
            duration: 1000,
            tokens: { input: 1000, output: 500, total: 1500 },
            cost: 0.01,
            success,
            timestamp: new Date().toISOString(),
        };
    }
    describe("calculateStringSimilarity", () => {
        it("returns 1 for identical strings", () => {
            expect(calculateStringSimilarity("hello world", "hello world")).toBe(1);
        });
        it("returns 0 for completely different strings", () => {
            expect(calculateStringSimilarity("hello", "world")).toBe(0);
        });
        it("returns partial similarity for overlapping words", () => {
            const sim = calculateStringSimilarity("SQL injection vulnerability", "SQL injection in login");
            expect(sim).toBeGreaterThan(0);
            expect(sim).toBeLessThan(1);
        });
        it("is case insensitive", () => {
            expect(calculateStringSimilarity("Hello World", "hello world")).toBe(1);
        });
        it("handles empty strings", () => {
            expect(calculateStringSimilarity("", "")).toBe(1);
            expect(calculateStringSimilarity("hello", "")).toBe(0);
        });
    });
    describe("calculateFindingSimilarity", () => {
        it("returns high score for identical findings", () => {
            const finding = createFinding({
                file: "src/auth.ts",
                line: 42,
                description: "SQL Injection",
                category: "security",
                severity: "high",
            });
            const sim = calculateFindingSimilarity(finding, finding);
            expect(sim).toBe(100);
        });
        it("returns high score for same file and line", () => {
            const a = createFinding({
                file: "src/auth.ts",
                line: 42,
                description: "SQL Injection vulnerability",
                category: "security",
            });
            const b = createFinding({
                file: "src/auth.ts",
                line: 42,
                description: "SQL Injection attack",
                category: "security",
            });
            const sim = calculateFindingSimilarity(a, b);
            expect(sim).toBeGreaterThan(70);
        });
        it("returns lower score for different files", () => {
            const a = createFinding({
                file: "src/auth.ts",
                line: 42,
                description: "SQL Injection",
                category: "security",
            });
            const b = createFinding({
                file: "src/user.ts",
                line: 100,
                description: "SQL Injection",
                category: "security",
            });
            const sim = calculateFindingSimilarity(a, b);
            expect(sim).toBeLessThan(70);
        });
        it("gives partial credit for nearby lines", () => {
            const a = createFinding({ file: "src/auth.ts", line: 42 });
            const b = createFinding({ file: "src/auth.ts", line: 45 });
            const c = createFinding({ file: "src/auth.ts", line: 100 });
            const simNear = calculateFindingSimilarity(a, b);
            const simFar = calculateFindingSimilarity(a, c);
            expect(simNear).toBeGreaterThan(simFar);
        });
    });
    describe("matchFindings", () => {
        it("matches identical findings across models", () => {
            const sharedFinding = createFinding({
                id: "shared",
                file: "src/auth.ts",
                line: 42,
                description: "SQL Injection",
                category: "security",
            });
            const results = [
                createModelResult("claude-3.5-sonnet", [{ ...sharedFinding, id: "claude-1" }]),
                createModelResult("gpt-4-turbo", [{ ...sharedFinding, id: "gpt-1" }]),
                createModelResult("gemini-1.5-pro", [{ ...sharedFinding, id: "gemini-1" }]),
            ];
            const matches = matchFindings(results);
            expect(matches.length).toBe(1);
            expect(matches[0].foundBy.length).toBe(3);
            expect(matches[0].matchType).toBe("exact");
            expect(matches[0].confidence).toBe(100);
        });
        it("identifies findings missed by some models", () => {
            const finding = createFinding({
                file: "src/auth.ts",
                line: 42,
                description: "SQL Injection",
            });
            const results = [
                createModelResult("claude-3.5-sonnet", [{ ...finding, id: "claude-1" }]),
                createModelResult("gpt-4-turbo", [{ ...finding, id: "gpt-1" }]),
                createModelResult("gemini-1.5-pro", []), // Gemini missed it
            ];
            const matches = matchFindings(results);
            expect(matches[0].foundBy).toContain("claude-3.5-sonnet");
            expect(matches[0].foundBy).toContain("gpt-4-turbo");
            expect(matches[0].missedBy).toContain("gemini-1.5-pro");
        });
        it("identifies unique findings from individual models", () => {
            const claudeFinding = createFinding({
                file: "src/auth.ts",
                line: 42,
                description: "Finding A",
            });
            const gptFinding = createFinding({
                file: "src/user.ts",
                line: 100,
                description: "Finding B",
            });
            const results = [
                createModelResult("claude-3.5-sonnet", [claudeFinding]),
                createModelResult("gpt-4-turbo", [gptFinding]),
            ];
            const matches = matchFindings(results);
            expect(matches.length).toBe(2);
            expect(matches.some((m) => m.foundBy.length === 1 && m.foundBy[0] === "claude-3.5-sonnet")).toBe(true);
            expect(matches.some((m) => m.foundBy.length === 1 && m.foundBy[0] === "gpt-4-turbo")).toBe(true);
        });
        it("handles empty results", () => {
            const results = [
                createModelResult("claude-3.5-sonnet", []),
                createModelResult("gpt-4-turbo", []),
            ];
            const matches = matchFindings(results);
            expect(matches).toEqual([]);
        });
        it("handles failed model runs", () => {
            const finding = createFinding();
            const results = [
                createModelResult("claude-3.5-sonnet", [finding]),
                { ...createModelResult("gpt-4-turbo", []), success: false, error: "API error" },
            ];
            const matches = matchFindings(results);
            expect(matches.length).toBe(1);
            // Failed model should not be counted
            expect(matches[0].foundBy).not.toContain("gpt-4-turbo");
        });
    });
    describe("findDisagreements", () => {
        it("identifies existence disagreements", () => {
            const finding = createFinding({
                file: "src/auth.ts",
                line: 42,
                severity: "high",
            });
            const results = [
                createModelResult("claude-3.5-sonnet", [{ ...finding, id: "claude-1" }]),
                createModelResult("gpt-4-turbo", [{ ...finding, id: "gpt-1" }]),
                createModelResult("gemini-1.5-pro", []),
            ];
            const matches = matchFindings(results);
            const disagreements = findDisagreements(matches, results);
            expect(disagreements.some((d) => d.type === "existence")).toBe(true);
        });
        it("identifies severity disagreements", () => {
            const baseFinding = createFinding({
                file: "src/auth.ts",
                line: 42,
                description: "SQL Injection",
                category: "security",
            });
            const results = [
                createModelResult("claude-3.5-sonnet", [{ ...baseFinding, id: "claude-1", severity: "critical" }]),
                createModelResult("gpt-4-turbo", [{ ...baseFinding, id: "gpt-1", severity: "high" }]),
                createModelResult("gemini-1.5-pro", [{ ...baseFinding, id: "gemini-1", severity: "medium" }]),
            ];
            const matches = matchFindings(results);
            const disagreements = findDisagreements(matches, results);
            expect(disagreements.some((d) => d.type === "severity")).toBe(true);
        });
        it("does not flag unanimous findings as disagreements", () => {
            const finding = createFinding({
                file: "src/auth.ts",
                line: 42,
                severity: "high",
            });
            const results = [
                createModelResult("claude-3.5-sonnet", [{ ...finding, id: "claude-1" }]),
                createModelResult("gpt-4-turbo", [{ ...finding, id: "gpt-1" }]),
                createModelResult("gemini-1.5-pro", [{ ...finding, id: "gemini-1" }]),
            ];
            const matches = matchFindings(results);
            const disagreements = findDisagreements(matches, results);
            // Should only have severity disagreement if severities differ
            expect(disagreements.filter((d) => d.type === "existence").length).toBe(0);
        });
    });
    describe("calculateFleissKappa", () => {
        it("returns high kappa for strong agreement", () => {
            // Create two findings that all models find
            const finding1 = createFinding({ file: "a.ts", line: 1, description: "Finding 1" });
            const finding2 = createFinding({ file: "b.ts", line: 2, description: "Finding 2" });
            const results = [
                createModelResult("claude-3.5-sonnet", [
                    { ...finding1, id: "c1" },
                    { ...finding2, id: "c2" },
                ]),
                createModelResult("gpt-4-turbo", [
                    { ...finding1, id: "g1" },
                    { ...finding2, id: "g2" },
                ]),
                createModelResult("gemini-1.5-pro", [
                    { ...finding1, id: "m1" },
                    { ...finding2, id: "m2" },
                ]),
            ];
            const matches = matchFindings(results);
            const kappa = calculateFleissKappa(matches, results);
            // When all models agree on all findings, kappa should be 1 (or close to it)
            expect(kappa).toBeGreaterThanOrEqual(0.8);
        });
        it("returns lower kappa for partial agreement", () => {
            const finding1 = createFinding({ file: "a.ts", line: 1, description: "Finding 1" });
            const finding2 = createFinding({ file: "b.ts", line: 2, description: "Finding 2" });
            const results = [
                createModelResult("claude-3.5-sonnet", [
                    { ...finding1, id: "c1" },
                    { ...finding2, id: "c2" },
                ]),
                createModelResult("gpt-4-turbo", [{ ...finding1, id: "g1" }]), // Missed finding2
                createModelResult("gemini-1.5-pro", [{ ...finding2, id: "m2" }]), // Missed finding1
            ];
            const matches = matchFindings(results);
            const kappa = calculateFleissKappa(matches, results);
            // Should be between -1 and 1
            expect(kappa).toBeGreaterThanOrEqual(-1);
            expect(kappa).toBeLessThanOrEqual(1);
            // Should be less than 1 due to disagreement
            expect(kappa).toBeLessThan(1);
        });
        it("handles single model results", () => {
            const finding = createFinding();
            const results = [
                createModelResult("claude-3.5-sonnet", [finding]),
            ];
            const matches = matchFindings(results);
            const kappa = calculateFleissKappa(matches, results);
            expect(kappa).toBe(0); // Can't calculate kappa with single rater
        });
        it("handles empty matches", () => {
            const results = [
                createModelResult("claude-3.5-sonnet", []),
                createModelResult("gpt-4-turbo", []),
            ];
            const matches = matchFindings(results);
            const kappa = calculateFleissKappa(matches, results);
            expect(kappa).toBe(0);
        });
    });
    describe("mergeFindings", () => {
        it("includes all findings when not requiring majority", () => {
            const finding1 = createFinding({ file: "a.ts", description: "Finding 1" });
            const finding2 = createFinding({ file: "b.ts", description: "Finding 2" });
            const results = [
                createModelResult("claude-3.5-sonnet", [{ ...finding1, id: "c1" }]),
                createModelResult("gpt-4-turbo", [{ ...finding2, id: "g2" }]),
            ];
            const matches = matchFindings(results);
            const merged = mergeFindings(matches, results, { requireMajority: false });
            expect(merged.length).toBe(2);
        });
        it("filters to majority when required", () => {
            const sharedFinding = createFinding({ file: "a.ts", description: "Shared Finding" });
            const uniqueFinding = createFinding({ file: "b.ts", description: "Unique Finding" });
            const results = [
                createModelResult("claude-3.5-sonnet", [
                    { ...sharedFinding, id: "c1" },
                    { ...uniqueFinding, id: "c2" },
                ]),
                createModelResult("gpt-4-turbo", [{ ...sharedFinding, id: "g1" }]),
                createModelResult("gemini-1.5-pro", [{ ...sharedFinding, id: "m1" }]),
            ];
            const matches = matchFindings(results);
            const merged = mergeFindings(matches, results, { requireMajority: true });
            // Only the shared finding should be included (found by 3/3 = 100% > 50%)
            expect(merged.length).toBe(1);
            expect(merged[0].description).toBe("Shared Finding");
        });
        it("uses highest severity when models disagree", () => {
            const baseFinding = createFinding({
                file: "a.ts",
                line: 1,
                description: "Finding",
                category: "security",
            });
            const results = [
                createModelResult("claude-3.5-sonnet", [{ ...baseFinding, id: "c1", severity: "critical" }]),
                createModelResult("gpt-4-turbo", [{ ...baseFinding, id: "g1", severity: "medium" }]),
            ];
            const matches = matchFindings(results);
            const merged = mergeFindings(matches, results);
            expect(merged[0].severity).toBe("critical");
        });
    });
    describe("calculateMultiModelConsensus", () => {
        it("calculates complete consensus result", () => {
            const finding = createFinding({
                file: "src/auth.ts",
                line: 42,
                description: "SQL Injection",
                category: "security",
                severity: "high",
            });
            const results = [
                createModelResult("claude-3.5-sonnet", [{ ...finding, id: "c1" }]),
                createModelResult("gpt-4-turbo", [{ ...finding, id: "g1" }]),
                createModelResult("gemini-1.5-pro", [{ ...finding, id: "m1" }]),
            ];
            const consensus = calculateMultiModelConsensus("cert-123", "security", results);
            expect(consensus.certificationId).toBe("cert-123");
            expect(consensus.agent).toBe("security");
            expect(consensus.models.length).toBe(3);
            expect(consensus.metrics.agreementRate).toBe(100);
            expect(consensus.metrics.unanimousFindings).toBe(1);
            expect(consensus.mergedFindings.length).toBe(1);
        });
        it("tracks metrics by severity", () => {
            const criticalFinding = createFinding({ severity: "critical", file: "a.ts", description: "Critical" });
            const highFinding = createFinding({ severity: "high", file: "b.ts", description: "High" });
            const results = [
                createModelResult("claude-3.5-sonnet", [
                    { ...criticalFinding, id: "c1" },
                    { ...highFinding, id: "c2" },
                ]),
                createModelResult("gpt-4-turbo", [
                    { ...criticalFinding, id: "g1" },
                    { ...highFinding, id: "g2" },
                ]),
            ];
            const consensus = calculateMultiModelConsensus("cert-123", "security", results);
            expect(consensus.metrics.bySeverity.critical.unanimous).toBe(1);
            expect(consensus.metrics.bySeverity.high.unanimous).toBe(1);
        });
    });
    describe("generateConsensusSummary", () => {
        it("generates markdown report", () => {
            const finding = createFinding();
            const results = [
                createModelResult("claude-3.5-sonnet", [finding]),
                createModelResult("gpt-4-turbo", [finding]),
            ];
            const consensus = calculateMultiModelConsensus("cert-123", "security", results);
            const summary = generateConsensusSummary(consensus);
            expect(summary).toContain("# Multi-Model Consensus Report");
            expect(summary).toContain("Agreement Rate");
            expect(summary).toContain("Fleiss' Kappa");
            expect(summary).toContain("| Severity |");
        });
        it("includes disagreements section when present", () => {
            const finding = createFinding({
                file: "src/auth.ts",
                line: 42,
                severity: "high",
            });
            const results = [
                createModelResult("claude-3.5-sonnet", [{ ...finding, id: "c1" }]),
                createModelResult("gpt-4-turbo", []),
            ];
            const consensus = calculateMultiModelConsensus("cert-123", "security", results);
            const summary = generateConsensusSummary(consensus);
            expect(summary).toContain("## Disagreements");
        });
    });
});
//# sourceMappingURL=consensus.test.js.map