const DEFAULT_TOP_K = 5;
export class ContextMappingEvaluator {
    ingestor;
    searcher;
    feedbackRecorder;
    relationCreator;
    constructor(ingestor, searcher, feedbackRecorder = isFeedbackRecorder(searcher) ? searcher : undefined, relationCreator = undefined) {
        this.ingestor = ingestor;
        this.searcher = searcher;
        this.feedbackRecorder = feedbackRecorder;
        this.relationCreator = relationCreator;
    }
    async run(fixture, options = {}) {
        const topK = options.topK ?? DEFAULT_TOP_K;
        const index = await this.seedKnowledge(fixture);
        await this.seedFeedback(fixture, index);
        await this.seedRelations(fixture, index);
        const cases = [];
        for (const testCase of fixture.cases) {
            cases.push(await this.evaluateCase(fixture, testCase, index, topK));
        }
        return {
            fixtureName: fixture.name,
            project: fixture.project,
            evaluatedAt: new Date().toISOString(),
            topK,
            totalCases: cases.length,
            metrics: buildMetrics(cases),
            cases,
        };
    }
    // ── Seeding ────────────────────────────────────────────────────────────
    async seedKnowledge(fixture) {
        const byEvalId = new Map();
        const byStoreId = new Map();
        const distractorIds = new Set();
        for (const item of fixture.knowledge) {
            const stored = await this.ingestOne(fixture.project, item, false);
            byEvalId.set(item.evalId, stored.id);
            byStoreId.set(stored.id, item.evalId);
        }
        for (const item of fixture.distractors ?? []) {
            const stored = await this.ingestOne(fixture.project, item, true);
            byEvalId.set(item.evalId, stored.id);
            byStoreId.set(stored.id, item.evalId);
            distractorIds.add(stored.id);
        }
        return { byEvalId, byStoreId, distractorIds };
    }
    async ingestOne(fixtureProject, item, isDistractor) {
        const { evalId, project, ...rest } = item;
        return this.ingestor.ingestKnowledge({
            ...rest,
            project: project ?? fixtureProject,
            metadata: {
                ...(rest.metadata ?? {}),
                evalId,
                contextMappingDistractor: isDistractor || undefined,
            },
        });
    }
    async seedFeedback(fixture, index) {
        if (!fixture.feedbackEvents?.length)
            return;
        if (!this.feedbackRecorder) {
            throw new Error('Context-mapping fixture defines feedbackEvents but no feedback recorder is configured.');
        }
        for (const event of fixture.feedbackEvents) {
            const rejectedKnowledgeIds = event.knowledgeIds
                ? [...resolveEvalIds(index, event.knowledgeIds)]
                : undefined;
            const contextPackId = event.prompt
                ? (await this.searcher.searchContext({
                    prompt: event.prompt,
                    project: event.project ?? fixture.project,
                    bypassCache: true,
                })).id
                : undefined;
            await this.feedbackRecorder.recordFeedback({
                contextPackId,
                project: event.project ?? fixture.project,
                feedbackType: event.feedbackType,
                reason: event.reason,
                rejectedKnowledgeIds,
                metadata: event.metadata,
            });
        }
    }
    async seedRelations(fixture, index) {
        if (!fixture.relations?.length)
            return;
        if (!this.relationCreator) {
            throw new Error('Context-mapping fixture defines relations but no relation creator is configured.');
        }
        for (const rel of fixture.relations) {
            const fromId = index.byEvalId.get(rel.fromEvalId);
            const toId = index.byEvalId.get(rel.toEvalId);
            if (!fromId || !toId) {
                throw new Error(`Unknown context-mapping relation evalIds: ${rel.fromEvalId} → ${rel.toEvalId}`);
            }
            await this.relationCreator.createKnowledgeRelation({
                project: fixture.project,
                fromKnowledgeId: fromId,
                relationType: rel.relationType,
                targetKind: rel.targetKind,
                targetKnowledgeId: toId,
                confidence: rel.confidence ?? 0.8,
                inferred: rel.inferred ?? false,
            });
        }
    }
    // ── Per-case evaluation ───────────────────────────────────────────────
    async evaluateCase(fixture, testCase, index, topK) {
        const expectedRelevant = resolveEvalIds(index, testCase.expectedRelevantKnowledgeIds ?? []);
        const directEvidence = resolveEvalIds(index, testCase.directEvidenceKnowledgeIds ?? []);
        const forbidden = resolveEvalIds(index, testCase.forbiddenKnowledgeIds ?? []);
        const noiseDistractors = resolveEvalIds(index, testCase.noiseDistractorIds ?? []);
        // Pack request mirrors RetrievalEvaluator so the metrics measure the same
        // pipeline path.
        const pack = await this.searcher.searchContext({
            prompt: testCase.prompt,
            project: testCase.project ?? fixture.project,
            taskType: testCase.taskType,
            files: testCase.files,
            symbols: testCase.symbols,
            errors: testCase.errors,
            tokenBudget: testCase.tokenBudget,
            bypassCache: true,
        });
        const allItems = flattenPack(pack);
        const topItems = allItems.slice(0, topK);
        const essentialItems = pack.sections.find((section) => section.name === 'essential')?.items ?? [];
        const topIds = new Set(topItems.map((item) => item.knowledgeId));
        const essentialIds = new Set(essentialItems.map((item) => item.knowledgeId));
        const allIds = new Set(allItems.map((item) => item.knowledgeId));
        // ── Metrics per case ───────────────────────────────────────────────
        // Context Precision @ k:  |topK ∩ expectedRelevant| / k.
        // Null when no expected set is declared.
        const contextPrecisionAtK = expectedRelevant.size === 0
            ? null
            : round(intersectionSize(topIds, expectedRelevant) / topK);
        // Context Recall: |expectedRelevant retrieved anywhere| / |expectedRelevant|.
        const contextRecall = expectedRelevant.size === 0
            ? null
            : round(intersectionSize(allIds, expectedRelevant) / expectedRelevant.size);
        // Context Entities Recall: of expected files+symbols, how many appear
        // somewhere in the pack candidates (content, labels, references)?
        const { recall: contextEntitiesRecall, missing: missingEntities } = evaluateEntityRecall(testCase.expectedEntities, allItems);
        // Direct-evidence placement: of direct IDs, how many landed in essential?
        const directEvidencePlacement = directEvidence.size === 0
            ? null
            : round(intersectionSize(essentialIds, directEvidence) / directEvidence.size);
        // Forbidden-item leakage count (in top-K) — used for forbidden-item rate
        // at the aggregate level.
        const forbiddenLeakageCount = intersectionSize(topIds, forbidden);
        // Noise-sensitivity scope: case-declared distractors take priority,
        // otherwise fall back to the global distractor pool so a fixture with no
        // per-case noise still gets a meaningful metric.
        const noiseScope = noiseDistractors.size > 0
            ? noiseDistractors
            : index.distractorIds;
        const distractorLeakageCount = intersectionSize(topIds, noiseScope);
        const noiseSensitivity = noiseScope.size === 0
            ? null
            : round(1 - distractorLeakageCount / noiseScope.size);
        // Fit calibration: per-case binary; aggregated into a rate below.
        const fitStatus = pack.contextFit?.fitStatus;
        const fitStatusPassed = testCase.expectedFitStatus === undefined
            ? true
            : fitStatus === testCase.expectedFitStatus;
        // Phase 8 — brief groundedness: every grounding-eligible action item in the assembled
        // brief must carry a non-empty evidenceIds list resolving to a pack candidate (or to
        // the action's own targetId for self-grounded review targets). Policy-only actions are
        // exempt. Target: 1.0 post-guard.
        const { score: briefGroundedness } = evaluateBriefGroundedness(pack, allItems);
        const briefWarnings = collectBriefWarnings(pack);
        const briefOk = briefGroundedness === null || briefGroundedness === 1;
        // Pass criteria for the case overall — used for human-readable PASS/FAIL.
        // A case passes when all *declared* expectations are satisfied. We don't
        // fail a case for an undeclared expectation.
        const precisionOk = contextPrecisionAtK === null || contextPrecisionAtK > 0;
        const recallOk = contextRecall === null || contextRecall === 1;
        const placementOk = directEvidencePlacement === null || directEvidencePlacement === 1;
        const noLeak = forbiddenLeakageCount === 0;
        const noDistractor = distractorLeakageCount === 0 || noiseDistractors.size === 0;
        const entityOk = contextEntitiesRecall === null || contextEntitiesRecall === 1;
        const passed = precisionOk && recallOk && placementOk && noLeak && noDistractor && entityOk && fitStatusPassed && briefOk;
        return {
            id: testCase.id,
            prompt: testCase.prompt,
            taxon: testCase.taxon,
            passed,
            fitStatus,
            expectedFitStatus: testCase.expectedFitStatus,
            fitStatusPassed,
            contextPrecisionAtK,
            contextRecall,
            contextEntitiesRecall,
            directEvidencePlacement,
            forbiddenLeakageCount,
            distractorLeakageCount,
            noiseSensitivity,
            briefGroundedness,
            topKnowledgeIds: toEvalIds(index, topItems.map((item) => item.knowledgeId)),
            essentialKnowledgeIds: toEvalIds(index, essentialItems.map((item) => item.knowledgeId)),
            expectedRelevantKnowledgeIds: toEvalIds(index, [...expectedRelevant]),
            directEvidenceKnowledgeIds: toEvalIds(index, [...directEvidence]),
            forbiddenKnowledgeIds: toEvalIds(index, [...forbidden]),
            missingEntities,
            briefWarnings,
        };
    }
}
// ────────────────────────────────────────────────────────────────────────
// Metrics + helpers
// ────────────────────────────────────────────────────────────────────────
function buildMetrics(cases) {
    const taxons = ['nl_to_code', 'code_to_code', 'text_to_text_doc', 'hybrid'];
    const perTaxon = taxons
        .map((taxon) => buildTaxonMetrics(taxon, cases.filter((testCase) => testCase.taxon === taxon)))
        .filter((entry) => entry !== null);
    return {
        contextPrecisionAtK: average(cases.map((testCase) => testCase.contextPrecisionAtK)),
        contextRecall: average(cases.map((testCase) => testCase.contextRecall)),
        contextEntitiesRecall: average(cases.map((testCase) => testCase.contextEntitiesRecall)),
        noiseSensitivity: average(cases.map((testCase) => testCase.noiseSensitivity)),
        directEvidencePlacement: average(cases.map((testCase) => testCase.directEvidencePlacement)),
        fitCalibration: average(fitCalibrationCases(cases)),
        forbiddenItemRate: forbiddenItemRate(cases),
        briefGroundedness: average(cases.map((testCase) => testCase.briefGroundedness)),
        perTaxon,
    };
}
function buildTaxonMetrics(taxon, cases) {
    if (cases.length === 0)
        return null;
    return {
        taxon,
        caseCount: cases.length,
        contextPrecisionAtK: average(cases.map((testCase) => testCase.contextPrecisionAtK)),
        contextRecall: average(cases.map((testCase) => testCase.contextRecall)),
        contextEntitiesRecall: average(cases.map((testCase) => testCase.contextEntitiesRecall)),
        noiseSensitivity: average(cases.map((testCase) => testCase.noiseSensitivity)),
        directEvidencePlacement: average(cases.map((testCase) => testCase.directEvidencePlacement)),
        fitCalibration: average(fitCalibrationCases(cases)),
        forbiddenItemRate: forbiddenItemRate(cases),
        briefGroundedness: average(cases.map((testCase) => testCase.briefGroundedness)),
    };
}
// Phase 8 — policy-only actions that are exempt from the groundedness guard. Keep in sync
// with the same set in `src/retrieval/context-pack.ts`.
const POLICY_ONLY_ACTIONS = new Set(['run_verification', 'ask_clarification', 'inspect_shortlist']);
function evaluateBriefGroundedness(pack, allItems) {
    const actionItems = pack.taskBrief?.actionItems ?? [];
    const eligible = actionItems.filter((action) => !POLICY_ONLY_ACTIONS.has(action.action));
    if (eligible.length === 0) {
        return { score: null, failingActions: [] };
    }
    const packIds = new Set(allItems.map((item) => item.knowledgeId));
    const failingActions = [];
    let grounded = 0;
    for (const action of eligible) {
        if (isActionGrounded(action, packIds)) {
            grounded += 1;
        }
        else {
            failingActions.push(`${action.action}:${action.targetPath ?? action.targetTitle ?? action.label}`);
        }
    }
    return {
        score: round(grounded / eligible.length),
        failingActions,
    };
}
function isActionGrounded(action, packIds) {
    if (POLICY_ONLY_ACTIONS.has(action.action)) {
        return true;
    }
    const evidenceIds = action.evidenceIds ?? [];
    if (evidenceIds.length === 0) {
        return false;
    }
    // Either every evidence id points to a pack candidate, or it points to the action's own
    // `targetId` (self-grounded review target — workbench resolves via the id directly).
    return evidenceIds.every((id) => packIds.has(id) || id === action.targetId);
}
function collectBriefWarnings(pack) {
    const notes = pack.contextFit?.fitDiagnostics?.notes ?? [];
    return notes.filter((note) => note.startsWith('brief_warning:'));
}
function fitCalibrationCases(cases) {
    return cases.map((testCase) => {
        if (testCase.expectedFitStatus === undefined)
            return null;
        return testCase.fitStatusPassed ? 1 : 0;
    });
}
function forbiddenItemRate(cases) {
    const withForbidden = cases.filter((testCase) => testCase.forbiddenKnowledgeIds.length > 0);
    if (withForbidden.length === 0)
        return null;
    const leaks = withForbidden.filter((testCase) => testCase.forbiddenLeakageCount > 0).length;
    return round(leaks / withForbidden.length);
}
function evaluateEntityRecall(expected, items) {
    if (!expected)
        return { recall: null, missing: [] };
    const entities = [...(expected.files ?? []), ...(expected.symbols ?? [])];
    if (entities.length === 0)
        return { recall: null, missing: [] };
    const haystack = items
        .map((item) => candidateHaystack(item))
        .join('\n')
        .toLowerCase();
    const missing = [];
    let matched = 0;
    for (const entity of entities) {
        if (haystack.includes(entity.toLowerCase())) {
            matched += 1;
        }
        else {
            missing.push(entity);
        }
    }
    return { recall: round(matched / entities.length), missing };
}
function candidateHaystack(item) {
    const labelText = item.labels.map((label) => `${label.type}:${label.value}`).join(' ');
    const refText = item.references.map((ref) => `${ref.type}:${ref.uri}`).join(' ');
    return [
        item.title,
        item.summary,
        item.content,
        item.contextualContent,
        labelText,
        refText,
    ].join(' ');
}
function flattenPack(pack) {
    return pack.sections.flatMap((section) => section.items);
}
function intersectionSize(left, right) {
    if (right.size === 0)
        return 0;
    let count = 0;
    for (const value of right) {
        if (left.has(value))
            count += 1;
    }
    return count;
}
function resolveEvalIds(index, evalIds) {
    return new Set(evalIds.map((evalId) => {
        const storeId = index.byEvalId.get(evalId);
        if (!storeId) {
            throw new Error(`Unknown context-mapping eval knowledge id: ${evalId}`);
        }
        return storeId;
    }));
}
function toEvalIds(index, storeIds) {
    return [...new Set(storeIds.map((storeId) => index.byStoreId.get(storeId) ?? storeId))];
}
function average(values) {
    const present = values.filter((value) => value !== null);
    if (present.length === 0)
        return null;
    return round(present.reduce((sum, value) => sum + value, 0) / present.length);
}
function round(value) {
    return Math.round(value * 10000) / 10000;
}
function isFeedbackRecorder(value) {
    return Boolean(value && typeof value === 'object' && 'recordFeedback' in value);
}
