const DEFAULT_TOP_K = 5;
export class RetrievalEvaluator {
    ingestor;
    searcher;
    feedbackRecorder;
    relationCreator;
    gapReader;
    atomCreator;
    statusUpdater;
    constructor(ingestor, searcher, feedbackRecorder = isFeedbackRecorder(searcher) ? searcher : undefined, relationCreator = undefined, gapReader = isKnowledgeGapReader(relationCreator) ? relationCreator : undefined, atomCreator = isAtomCreator(relationCreator) ? relationCreator : undefined, statusUpdater = isKnowledgeStatusUpdater(relationCreator) ? relationCreator : undefined) {
        this.ingestor = ingestor;
        this.searcher = searcher;
        this.feedbackRecorder = feedbackRecorder;
        this.relationCreator = relationCreator;
        this.gapReader = gapReader;
        this.atomCreator = atomCreator;
        this.statusUpdater = statusUpdater;
    }
    async run(fixture, options = {}) {
        const topK = options.topK ?? DEFAULT_TOP_K;
        const index = await this.seedKnowledge(fixture);
        await this.seedAtoms(fixture, index);
        await this.seedFeedback(fixture, index);
        await this.seedRelations(fixture, index);
        const cases = [];
        for (const testCase of fixture.cases) {
            cases.push(await this.evaluateCase(fixture, testCase, index, topK));
        }
        return {
            fixtureName: fixture.name,
            project: fixture.project,
            evaluatedAt: new Date().toISOString(),
            topK,
            totalCases: cases.length,
            metrics: buildMetrics(cases, topK),
            cases,
        };
    }
    async seedKnowledge(fixture) {
        const byEvalId = new Map();
        const byStoreId = new Map();
        for (const item of fixture.knowledge) {
            const { evalId, project, status, ...knowledge } = item;
            const stored = await this.ingestor.ingestKnowledge({
                ...knowledge,
                project: project ?? fixture.project,
                metadata: {
                    ...(knowledge.metadata ?? {}),
                    evalId,
                },
            });
            // Simulate a deleted/archived source file: flip status after ingest (ingest always
            // creates approved). Archived items must drop out of retrieval entirely.
            if (status && status !== 'approved' && this.statusUpdater) {
                await this.statusUpdater.updateKnowledge(stored.id, { status });
            }
            byEvalId.set(evalId, stored.id);
            byStoreId.set(stored.id, evalId);
        }
        return { byEvalId, byStoreId };
    }
    async seedAtoms(fixture, index) {
        if (!fixture.atoms?.length) {
            return;
        }
        if (!this.atomCreator) {
            throw new Error('Retrieval eval fixture defines atoms but no atom creator is configured.');
        }
        for (const atom of fixture.atoms) {
            const created = await this.atomCreator.createAtom({
                project: atom.project ?? fixture.project,
                claim: atom.claim,
                type: atom.type,
                evidence: atom.evidence,
                trigger: atom.trigger,
                verification: atom.verification,
                producedBy: atom.producedBy ?? 'agent_session',
                scope: atom.scope,
                teamId: atom.teamId,
            });
            if (atom.tier || atom.reuseCount !== undefined || atom.lastReusedAt || atom.status) {
                await this.atomCreator.updateAtom(created.id, {
                    tier: atom.tier,
                    reuseCount: atom.reuseCount,
                    lastReusedAt: atom.lastReusedAt,
                    status: atom.status,
                });
            }
            index.byEvalId.set(atom.evalId, created.id);
            index.byStoreId.set(created.id, atom.evalId);
        }
    }
    async seedFeedback(fixture, index) {
        if (!fixture.feedbackEvents?.length) {
            return;
        }
        if (!this.feedbackRecorder) {
            throw new Error('Retrieval eval fixture defines feedbackEvents but no feedback recorder is configured.');
        }
        for (const event of fixture.feedbackEvents) {
            const rejectedKnowledgeIds = event.knowledgeIds
                ? [...resolveEvalIds(index, event.knowledgeIds)]
                : undefined;
            const contextPackId = event.prompt
                ? (await this.searcher.searchContext({
                    prompt: event.prompt,
                    project: event.project ?? fixture.project,
                    bypassCache: true,
                })).id
                : undefined;
            await this.feedbackRecorder.recordFeedback({
                contextPackId,
                project: event.project ?? fixture.project,
                feedbackType: event.feedbackType,
                reason: event.reason,
                rejectedKnowledgeIds,
                metadata: event.metadata,
            });
            if (event.expectedKnowledgeGap) {
                await this.assertExpectedKnowledgeGap(fixture, event, contextPackId);
            }
        }
    }
    async assertExpectedKnowledgeGap(fixture, event, contextPackId) {
        if (!this.gapReader) {
            throw new Error('Retrieval eval fixture expects knowledge gaps but no gap reader is configured.');
        }
        const expected = event.expectedKnowledgeGap;
        if (!expected) {
            return;
        }
        const gaps = await this.gapReader.listKnowledgeGaps({
            project: event.project ?? fixture.project,
            status: expected.status ?? 'open',
            contextPackId,
            limit: 20,
        });
        const matchingGap = gaps.find((gap) => knowledgeGapMatches(gap, expected, contextPackId));
        if (!matchingGap) {
            throw new Error(`Expected missing_context feedback to create a matching knowledge gap for prompt: ${event.prompt ?? event.reason ?? fixture.name}`);
        }
    }
    async seedRelations(fixture, index) {
        if (!fixture.relations?.length) {
            return;
        }
        if (!this.relationCreator) {
            throw new Error('Retrieval eval fixture defines relations but no relation creator is configured.');
        }
        for (const rel of fixture.relations) {
            const fromId = index.byEvalId.get(rel.fromEvalId);
            const toId = index.byEvalId.get(rel.toEvalId);
            if (!fromId || !toId) {
                throw new Error(`Unknown eval relation evalIds: ${rel.fromEvalId} → ${rel.toEvalId}`);
            }
            await this.relationCreator.createKnowledgeRelation({
                project: fixture.project,
                fromKnowledgeId: fromId,
                relationType: rel.relationType,
                targetKind: rel.targetKind,
                targetKnowledgeId: toId,
                confidence: rel.confidence ?? 0.8,
                inferred: rel.inferred ?? false,
            });
        }
    }
    async evaluateCase(fixture, testCase, index, topK) {
        const expectedIds = resolveEvalIds(index, testCase.expectedKnowledgeIds ?? []);
        const expectedSelectedIds = resolveEvalIds(index, testCase.expectedSelectedKnowledgeIds ?? []);
        const unexpectedIds = resolveEvalIds(index, testCase.unexpectedKnowledgeIds ?? []);
        const rejectedIds = resolveEvalIds(index, testCase.rejectedKnowledgeIds ?? []);
        const pack = await this.searcher.searchContext({
            prompt: testCase.prompt,
            project: testCase.project ?? fixture.project,
            taskType: testCase.taskType,
            files: testCase.files,
            symbols: testCase.symbols,
            errors: testCase.errors,
            noiseTolerance: testCase.noiseTolerance,
            tokenBudget: testCase.tokenBudget,
            rejectedKnowledgeIds: [...rejectedIds],
            bypassCache: true,
        });
        const rankedItems = flattenPack(pack);
        const topItems = rankedItems.slice(0, topK);
        const selectedIds = rankedItems.map((item) => item.knowledgeId);
        const firstExpectedRank = firstRank(rankedItems, expectedIds);
        const matchedExpectedIds = topItems
            .filter((item) => expectedIds.has(item.knowledgeId))
            .map((item) => item.knowledgeId);
        const returnedUnexpectedIds = topItems
            .filter((item) => unexpectedIds.has(item.knowledgeId))
            .map((item) => item.knowledgeId);
        const returnedRejectedIds = rankedItems
            .filter((item) => rejectedIds.has(item.knowledgeId))
            .map((item) => item.knowledgeId);
        const matchedSelectedIds = selectedIds.filter((knowledgeId) => expectedSelectedIds.has(knowledgeId));
        const classificationChecks = evaluateClassification(testCase.expectedClassification, pack.classified);
        const hitPassed = expectedIds.size === 0 || (firstExpectedRank !== null && firstExpectedRank <= topK);
        const selectedPassed = expectedSelectedIds.size === 0 || matchedSelectedIds.length === expectedSelectedIds.size;
        const confidencePassed = testCase.minConfidence === undefined || pack.confidence >= testCase.minConfidence;
        const contextFitStatusPassed = testCase.expectedContextFitStatus === undefined
            || pack.contextFit?.fitStatus === testCase.expectedContextFitStatus;
        const contextFitScorePassed = testCase.minContextFitScore === undefined
            || (pack.contextFit?.fitScore ?? 0) >= testCase.minContextFitScore;
        const classificationPassed = classificationChecks.every((check) => check.passed);
        const passed = hitPassed
            && selectedPassed
            && returnedUnexpectedIds.length === 0
            && returnedRejectedIds.length === 0
            && confidencePassed
            && contextFitStatusPassed
            && contextFitScorePassed
            && classificationPassed;
        return {
            id: testCase.id,
            prompt: testCase.prompt,
            passed,
            confidence: pack.confidence,
            minConfidence: testCase.minConfidence,
            confidencePassed,
            contextFitStatus: pack.contextFit?.fitStatus,
            expectedContextFitStatus: testCase.expectedContextFitStatus,
            contextFitStatusPassed,
            contextFitScore: pack.contextFit?.fitScore,
            minContextFitScore: testCase.minContextFitScore,
            contextFitScorePassed,
            reciprocalRank: firstExpectedRank === null ? null : round(1 / firstExpectedRank),
            precisionAtK: expectedIds.size === 0 ? null : round(matchedExpectedIds.length / topK),
            firstExpectedRank,
            expectedKnowledgeIds: toEvalIds(index, [...expectedIds]),
            matchedExpectedKnowledgeIds: toEvalIds(index, matchedExpectedIds),
            expectedSelectedKnowledgeIds: toEvalIds(index, [...expectedSelectedIds]),
            matchedSelectedKnowledgeIds: toEvalIds(index, matchedSelectedIds),
            unexpectedKnowledgeIds: toEvalIds(index, [...unexpectedIds]),
            returnedUnexpectedKnowledgeIds: toEvalIds(index, returnedUnexpectedIds),
            rejectedKnowledgeIds: toEvalIds(index, [...rejectedIds]),
            returnedRejectedKnowledgeIds: toEvalIds(index, returnedRejectedIds),
            selectedKnowledgeIds: toEvalIds(index, selectedIds),
            topKnowledgeIds: toEvalIds(index, topItems.map((item) => item.knowledgeId)),
            classificationChecks,
        };
    }
}
function flattenPack(pack) {
    return pack.sections.flatMap((section) => section.items);
}
function firstRank(items, expectedIds) {
    if (expectedIds.size === 0) {
        return null;
    }
    const index = items.findIndex((item) => expectedIds.has(item.knowledgeId));
    return index === -1 ? null : index + 1;
}
function resolveEvalIds(index, evalIds) {
    return new Set(evalIds.map((evalId) => {
        const storeId = index.byEvalId.get(evalId);
        if (!storeId) {
            throw new Error(`Unknown retrieval eval knowledge id: ${evalId}`);
        }
        return storeId;
    }));
}
function toEvalIds(index, storeIds) {
    return [...new Set(storeIds.map((storeId) => index.byStoreId.get(storeId) ?? storeId))];
}
function evaluateClassification(expected, actual) {
    if (!expected) {
        return [];
    }
    return [
        arrayCheck('files', expected.files, actual.files),
        arrayCheck('symbols', expected.symbols, actual.symbols),
        arrayCheck('errors', expected.errors, actual.errors),
        arrayCheck('technologies', expected.technologies, actual.technologies),
        arrayCheck('businessAreas', expected.businessAreas, actual.businessAreas),
        scalarCheck('taskType', expected.taskType, actual.taskType),
    ].filter((check) => Boolean(check));
}
function arrayCheck(field, expected, actual) {
    if (expected === undefined) {
        return undefined;
    }
    return {
        field,
        expected,
        actual,
        passed: expected.length === 0 ? actual.length === 0 : containsAll(actual, expected),
    };
}
function scalarCheck(field, expected, actual) {
    if (!expected) {
        return undefined;
    }
    return {
        field,
        expected: [expected],
        actual: [actual],
        passed: expected === actual,
    };
}
function containsAll(actual, expected) {
    const actualValues = new Set(actual);
    return expected.every((value) => actualValues.has(value));
}
function buildMetrics(cases, topK) {
    const casesWithExpected = cases.filter((testCase) => testCase.expectedKnowledgeIds.length > 0);
    const casesWithSelected = cases.filter((testCase) => testCase.expectedSelectedKnowledgeIds.length > 0);
    const casesWithRejected = cases.filter((testCase) => testCase.rejectedKnowledgeIds.length > 0);
    const casesWithUnexpected = cases.filter((testCase) => testCase.unexpectedKnowledgeIds.length > 0);
    const confidenceCases = cases.filter((testCase) => testCase.minConfidence !== undefined);
    const contextFitStatusCases = cases.filter((testCase) => testCase.expectedContextFitStatus !== undefined);
    const contextFitScoreCases = cases.filter((testCase) => testCase.minContextFitScore !== undefined);
    const classificationCases = cases.filter((testCase) => testCase.classificationChecks.length > 0);
    return {
        hitRate: average(casesWithExpected.map((testCase) => (testCase.firstExpectedRank !== null && testCase.firstExpectedRank <= topK ? 1 : 0))),
        meanReciprocalRank: average(casesWithExpected.map((testCase) => testCase.reciprocalRank ?? 0)),
        precisionAtK: average(casesWithExpected.map((testCase) => testCase.precisionAtK ?? 0)),
        selectedCoverageRate: average(casesWithSelected.map((testCase) => (testCase.matchedSelectedKnowledgeIds.length === testCase.expectedSelectedKnowledgeIds.length ? 1 : 0))),
        staleRejectionRate: average(casesWithRejected.map((testCase) => (testCase.returnedRejectedKnowledgeIds.length === 0 ? 1 : 0))),
        unexpectedAvoidanceRate: average(casesWithUnexpected.map((testCase) => (testCase.returnedUnexpectedKnowledgeIds.length === 0 ? 1 : 0))),
        confidenceThresholdRate: average(confidenceCases.map((testCase) => (testCase.confidencePassed ? 1 : 0))),
        contextFitStatusRate: average(contextFitStatusCases.map((testCase) => (testCase.contextFitStatusPassed ? 1 : 0))),
        contextFitScoreRate: average(contextFitScoreCases.map((testCase) => (testCase.contextFitScorePassed ? 1 : 0))),
        exactFileMatchRate: checkRate(cases, 'files'),
        exactSymbolMatchRate: checkRate(cases, 'symbols'),
        exactErrorMatchRate: checkRate(cases, 'errors'),
        exactClassificationMatchRate: average(classificationCases.map((testCase) => (testCase.classificationChecks.every((check) => check.passed) ? 1 : 0))),
    };
}
function checkRate(cases, field) {
    const checks = cases.flatMap((testCase) => testCase.classificationChecks.filter((check) => check.field === field));
    return average(checks.map((check) => (check.passed ? 1 : 0)));
}
function average(values) {
    if (values.length === 0) {
        return null;
    }
    return round(values.reduce((sum, value) => sum + value, 0) / values.length);
}
function round(value) {
    return Math.round(value * 10000) / 10000;
}
function isFeedbackRecorder(value) {
    return Boolean(value && typeof value === 'object' && 'recordFeedback' in value);
}
function isKnowledgeGapReader(value) {
    return Boolean(value && typeof value === 'object' && 'listKnowledgeGaps' in value);
}
function isAtomCreator(value) {
    return Boolean(value && typeof value === 'object' && 'createAtom' in value && 'updateAtom' in value);
}
function isKnowledgeStatusUpdater(value) {
    return Boolean(value && typeof value === 'object' && 'updateKnowledge' in value);
}
function knowledgeGapMatches(gap, expected, contextPackId) {
    if (expected.contextPackIdRequired !== false && (!contextPackId || gap.contextPackId !== contextPackId)) {
        return false;
    }
    if (expected.promptIncludes && !gap.prompt.toLowerCase().includes(expected.promptIncludes.toLowerCase())) {
        return false;
    }
    if (expected.reasonIncludes && !gap.reason?.toLowerCase().includes(expected.reasonIncludes.toLowerCase())) {
        return false;
    }
    if (expected.missingSignals && !containsAll(gap.missingSignals, expected.missingSignals)) {
        return false;
    }
    return true;
}
