// Epic #189 Slice 2 — heterogeneous grounded merge. A chat may carry BOTH connected folders
// (#532, lexical) AND Local Knowledge connectors (#189, vector), or two or more connectors. Asking
// one question must retrieve from EVERY source and return ONE merged grounded answer with
// source-tagged citations from both engines. This module owns that merge branch only; the
// folders-only (#532) and single-connector (#189) paths are untouched so their wire output stays
// byte-identical (AC). It composes the exported folder helpers (grounded-qa-multi-source.ts) and
// connector seams (local-knowledge-grounded-qa.ts) without re-implementing retrieval.
import { randomUUID } from "node:crypto";
import { resolveCostClass } from "@oscharko-dev/keiko-model-gateway";
import { persistConnectedContextEvidence } from "@oscharko-dev/keiko-evidence";
import { createSqliteAuditSink, readCitationExcerpt, runLocalKnowledgeRetrieval, } from "@oscharko-dev/keiko-local-knowledge";
import { rerankAndSelect } from "./grounded-rerank.js";
import { CANDIDATE_OMISSION_REASONS, CONNECTED_CONTEXT_SCHEMA_VERSION, DEFAULT_EXPLORATION_BUDGET, } from "@oscharko-dev/keiko-contracts/connected-context";
import { buildGroundedAnswerContextPackSummary, } from "@oscharko-dev/keiko-contracts/bff-wire";
import { redact } from "@oscharko-dev/keiko-security";
import { errorBody } from "./routes.js";
import { currentGroundingLimits, currentRedactionSecrets } from "./deps.js";
import {} from "./grounded-orchestrator.js";
import { buildConnectedScopes, defaultRetriever, mergeContextPackSummaries, sourceLabels, splitExplorationBudget, } from "./grounded-qa-multi-source.js";
import { MAX_PROMPT_REFERENCES, buildSelectedScopeSourceLookup, createEmbeddingAdapter, openStoreForDeps, projectLocalKnowledgeCitation, scopeStateFailure, selectedCapsulesForScope, } from "./local-knowledge-grounded-qa.js";
import { GROUNDED_SYSTEM_PROMPT } from "./grounded-prompt.js";
import { normalizeGroundedAnswerPayload, } from "./grounded-answer.js";
import { buildCitations, buildQuery, buildSelectedScopeFrom, deriveScopeIdFrom, ensureNotCancelled, internalError, isValidGroundedPack, mappedGatewayError, persistGroundedExchange, redactString, } from "./grounded-qa.js";
// ─── Canonical connector reader ───────────────────────────────────────────────
// Mirrors buildConnectedScopes: the plural `localKnowledgeScopes` list supersedes the legacy single
// `localKnowledgeScope`. Readers must not mix the two — the list, when present, is authoritative.
export function buildLocalKnowledgeScopes(chat) {
    return chat.localKnowledgeScopes ?? (chat.localKnowledgeScope ? [chat.localKnowledgeScope] : []);
}
// ─── Connector source labels (disambiguated like sourceLabels) ────────────────
export function connectorLabels(rawLabels) {
    const counts = new Map();
    for (const raw of rawLabels)
        counts.set(raw, (counts.get(raw) ?? 0) + 1);
    const seen = new Map();
    return rawLabels.map((raw) => {
        if ((counts.get(raw) ?? 0) <= 1)
            return raw;
        const ordinal = (seen.get(raw) ?? 0) + 1;
        seen.set(raw, ordinal);
        return `${raw}#${String(ordinal)}`;
    });
}
// Builds a single RRF-selected set that covers both folder and connector candidates. The selected
// set is the SOLE source of truth for both the prompt and the citations; the two paths must not
// diverge from this point forward.
function folderRerankInputs(folders, redactor) {
    return folders.flatMap((src) => src.pack.files.flatMap((file) => file.excerpts.map((excerpt) => ({
        kind: "folder",
        redactedText: redactString(redactor, excerpt.content),
        engineScore: excerpt.atom.score,
        sourceLabel: src.label,
        tieKey: excerpt.atom.stableId,
        payload: {
            kind: "folder",
            scopePath: excerpt.atom.scopePath,
            lineRange: excerpt.atom.lineRange,
            score: excerpt.atom.score,
            stableId: excerpt.atom.stableId,
        },
    }))));
}
function connectorRerankInputs(connectors, store, redactor, maxPromptReferences, maxExcerptChars) {
    return connectors.flatMap((src) => {
        const lookup = buildSelectedScopeSourceLookup(store, src.selected);
        return src.references.slice(0, maxPromptReferences).map((reference) => ({
            kind: "connector",
            redactedText: redactString(redactor, readCitationExcerpt(store, reference.capsuleId, reference.citation, maxExcerptChars)),
            engineScore: reference.score,
            sourceLabel: src.label,
            tieKey: String(reference.chunkId),
            payload: { kind: "connector", reference, lookup },
        }));
    });
}
function buildUnifiedSelection(ctx, folders, connectors, store) {
    const limits = currentGroundingLimits(ctx.deps);
    const { redactor } = ctx.deps;
    const inputs = [
        ...folderRerankInputs(folders, redactor),
        ...connectorRerankInputs(connectors, store, redactor, limits.maxPromptReferences, limits.maxExcerptChars),
    ];
    return rerankAndSelect(inputs, {
        maxCandidates: limits.hybridMaxCandidates,
        maxExcerptBytes: limits.hybridMaxExcerptBytes,
    });
}
// ─── Folder retrieval (mirrors runMultiSourceAsk's loop) ──────────────────────
async function retrieveFolderPacks(ctx, folderScopes, query, retriever) {
    const labels = sourceLabels(folderScopes);
    const budget = splitExplorationBudget(DEFAULT_EXPLORATION_BUDGET, folderScopes.length);
    const retrieved = [];
    for (let i = 0; i < folderScopes.length; i += 1) {
        ensureNotCancelled(ctx.signal);
        const cs = folderScopes[i];
        const label = labels[i];
        if (cs === undefined || label === undefined)
            continue;
        const scope = buildSelectedScopeFrom(ctx.chat, cs, deriveScopeIdFrom(ctx.chat, cs, i));
        const out = await retriever({
            scope,
            query,
            workspaceRoot: scope.workspaceRoot,
            budget,
        });
        if (!isValidGroundedPack(out.pack)) {
            return internalError("Grounded answer context pack failed validation.");
        }
        retrieved.push({ label, pack: out.pack, elapsedMs: out.elapsedMs, scope });
    }
    return retrieved;
}
// ─── Connector retrieval ──────────────────────────────────────────────────────
function resolveConnectorScopes(connectorScopes, store) {
    const resolved = [];
    for (const scope of connectorScopes) {
        const selected = selectedCapsulesForScope(scope, store);
        if ("status" in selected)
            return selected;
        resolved.push(selected);
    }
    return resolved;
}
function distinctEmbeddingModelIds(selected) {
    return Array.from(new Set(selected.capsules.map((capsule) => capsule.embeddingModelIdentity.modelId)));
}
function connectorQuery(scope, content) {
    return {
        text: content,
        topK: MAX_PROMPT_REFERENCES,
        ...(scope.kind === "capsule" ? { capsuleId: scope.capsuleId } : {}),
        ...(scope.kind === "capsule-set" ? { capsuleSetId: scope.capsuleSetId } : {}),
    };
}
function defaultConnectorRetrieve(ctx) {
    return async (store, scope, selected) => {
        const embeddingAdapter = createEmbeddingAdapter(ctx.deps, distinctEmbeddingModelIds(selected));
        if ("status" in embeddingAdapter) {
            throw new EmbeddingAdapterError(embeddingAdapter);
        }
        return runLocalKnowledgeRetrieval({ store, embeddingAdapter, signal: ctx.signal }, connectorQuery(scope, ctx.content));
    };
}
class EmbeddingAdapterError extends Error {
    result;
    constructor(result) {
        super("embedding adapter unavailable");
        this.result = result;
        this.name = "EmbeddingAdapterError";
    }
}
async function retrieveConnectors(ctx, store, connectorScopes, resolved) {
    const retrieve = ctx.connectorRetrieve ?? defaultConnectorRetrieve(ctx);
    const labels = connectorLabels(resolved.map((s) => s.scopeLabel));
    const retrieved = [];
    const skipped = [];
    for (let i = 0; i < connectorScopes.length; i += 1) {
        ensureNotCancelled(ctx.signal);
        const scope = connectorScopes[i];
        const selected = resolved[i];
        const label = labels[i];
        if (scope === undefined || selected === undefined || label === undefined)
            continue;
        const failure = scopeStateFailure(selected);
        if (failure !== undefined) {
            skipped.push({ label, reason: failure.reason, message: failure.message });
            continue;
        }
        const outcome = await retrieveOneConnector(retrieve, store, scope, selected);
        if ("status" in outcome)
            return outcome;
        retrieved.push({ label, selected, references: outcome.references });
    }
    return { retrieved, skipped };
}
async function retrieveOneConnector(retrieve, store, scope, selected) {
    try {
        return await retrieve(store, scope, selected);
    }
    catch (error) {
        if (error instanceof EmbeddingAdapterError)
            return error.result;
        throw error;
    }
}
// ─── Merged prompt ────────────────────────────────────────────────────────────
const HYBRID_SYSTEM_PROMPT = `${GROUNDED_SYSTEM_PROMPT} Connector excerpts are indexed-document citations: attribute every ` +
    "connector claim to its source label and the matching [n] marker in addition to any file reference.";
// Builds the user message from the SAME selected set used for citations. Each candidate gets a
// single global [n] marker that is consistent with the citation arrays. redactedText is
// already redacted — do NOT pass it through redactString again.
function buildRerankedHybridUserMessage(question, selected, redactor) {
    const folderCount = selected.filter((s) => s.kind === "folder").length;
    const connectorCount = selected.filter((s) => s.kind === "connector").length;
    const lines = [
        "User question:",
        redactString(redactor, question),
        "",
        `Connected sources: ${String(folderCount)} folder(s), ${String(connectorCount)} connector(s).`,
        "Cite every claim by its [n] marker and source label.",
        "",
    ];
    for (const candidate of selected) {
        const kindLabel = candidate.kind === "folder" ? "Folder" : "Connector";
        lines.push(`[${String(candidate.marker)}] ### ${kindLabel} source: ${candidate.sourceLabel}`);
        lines.push("```text");
        lines.push(candidate.redactedText.length > 0 ? candidate.redactedText : "(No excerpt text available.)");
        lines.push("```");
        lines.push("");
    }
    return lines.join("\n");
}
export function createHybridAnswerer(model, modelId, signal) {
    return async (system, user) => {
        ensureNotCancelled(signal);
        const response = await model.call({
            modelId,
            messages: [
                { role: "system", content: system },
                { role: "user", content: user },
            ],
            stream: false,
        }, signal);
        const content = response.content.trim();
        return {
            content: content.length > 0 ? content : "The model returned an empty response.",
            usage: {
                promptTokens: response.usage.promptTokens,
                completionTokens: response.usage.completionTokens,
            },
        };
    };
}
// ─── Citations + summaries ────────────────────────────────────────────────────
// Both citation arrays are derived from the SAME selected set so that the [n] markers in the
// prompt are always consistent with the citation arrays surfaced to the client.
function selectedFolderCitations(selected, redactor) {
    return selected
        .filter((s) => s.kind === "folder")
        .map((s) => ({
        scopePath: redactString(redactor, s.payload.scopePath),
        lineRange: s.payload.lineRange,
        score: s.payload.score,
        stableId: redactString(redactor, s.payload.stableId),
        source: s.sourceLabel,
        marker: s.marker,
    }));
}
function selectedConnectorCitations(selected) {
    return selected
        .filter((s) => s.kind === "connector")
        .map((s) => projectLocalKnowledgeCitation(s.payload.reference, `[${String(s.marker)}]`, s.payload.lookup));
}
function zeroExploration() {
    return {
        searchCalls: 0,
        filesRead: 0,
        excerptBytes: 0,
        modelInputTokens: 0,
        modelOutputTokens: 0,
        elapsedMs: 0,
        rerankCalls: 0,
    };
}
function zeroBudget() {
    return {
        searchCallsMax: 0,
        filesReadMax: 0,
        excerptBytesMax: 0,
        modelInputTokensMax: 0,
        modelOutputTokensMax: 0,
        elapsedMsMax: 0,
        rerankCallsMax: 0,
    };
}
function zeroOmittedCounts() {
    const counts = {};
    for (const reason of CANDIDATE_OMISSION_REASONS)
        counts[reason] = 0;
    return counts;
}
// The hybrid contract requires a folder summary even when a chat has zero folders (connector-only
// merge). This is a structurally empty, deterministic summary — no source pack to derive from.
function emptyFolderSummary() {
    return {
        schemaVersion: CONNECTED_CONTEXT_SCHEMA_VERSION,
        scopeId: "scope-empty",
        scopeKind: "workspace-root",
        fileCount: 0,
        queryKind: "natural-language",
        usage: zeroExploration(),
        budget: zeroBudget(),
        citationCount: 0,
        omittedCount: 0,
        omittedCounts: zeroOmittedCounts(),
        uncertaintyCount: 0,
        elapsedMs: 0,
    };
}
function folderSummary(folders, redactor) {
    if (folders.length === 0)
        return emptyFolderSummary();
    return mergeContextPackSummaries(folders.map((src) => buildGroundedAnswerContextPackSummary(src.pack, buildCitations(src.pack, redactor).length, src.elapsedMs)));
}
function hashString32(value) {
    let hash = 0x811c9dc5;
    for (let i = 0; i < value.length; i += 1) {
        hash ^= value.charCodeAt(i);
        hash = Math.imul(hash, 0x01000193);
    }
    return (hash >>> 0).toString(16).padStart(8, "0");
}
function connectorSourceCount(connectors) {
    const sourceIds = new Set();
    for (const src of connectors) {
        for (const capsule of src.selected.capsules) {
            for (const id of capsule.sourceIds)
                sourceIds.add(String(id));
        }
    }
    return sourceIds.size;
}
// One merged knowledge summary across every connector. Counts are aggregated so the wire shape
// stays a single LocalKnowledgeGroundedAnswerContextSummary even when N connectors contributed.
// referencesUsed = connector candidates in the SHARED selected set; referenceBudget = the shared
// hybridMaxCandidates cap so the invariant referencesUsed <= referenceBudget always holds.
function knowledgeSummary(chat, connectors, citationCount, referencesUsed, referenceBudget) {
    const capsuleCount = connectors.reduce((acc, src) => acc + src.selected.capsules.length, 0);
    const label = connectors.map((src) => src.label).join("+");
    const scopeKind = connectors.length === 1 ? connectors[0]?.selected.scopeKind : "capsule-set";
    return {
        kind: "local-knowledge",
        scopeKind: scopeKind ?? "capsule-set",
        scopeId: `lk-${hashString32(`${chat.id}|${label}`)}`,
        scopeLabel: label,
        capsuleCount,
        sourceCount: connectorSourceCount(connectors),
        citationCount,
        referenceBudget,
        referencesUsed,
    };
}
function skippedUncertainty(skipped, redactor) {
    return skipped.map((entry) => ({
        kind: entry.reason,
        claim: redactString(redactor, `Connector ${entry.label} skipped: ${entry.message}`),
    }));
}
function folderUncertainty(folders, redactor) {
    return folders.flatMap((src) => src.pack.uncertainty.map((u) => ({
        kind: u.kind,
        claim: redactString(redactor, u.claim),
    })));
}
// ─── Evidence persistence ─────────────────────────────────────────────────────
// Persists ONE evidence run per folder source (mirrors the #532 per-source persist) plus the
// connector retrieval/answer-context audit via the LK sink (mirrors the single-connector path).
// Returns the first folder run id, surfaced as the answer's primary evidenceRunId.
function persistFolderEvidence(ctx, folders) {
    let firstRunId;
    for (const src of folders) {
        const finishedAt = Date.now();
        const startedAt = Math.max(0, finishedAt - src.elapsedMs);
        const runId = `grounded-${randomUUID()}`;
        persistConnectedContextEvidence({
            runId,
            modelId: ctx.modelId,
            workspaceRoot: src.scope.workspaceRoot,
            chatId: ctx.chat.id,
            pack: src.pack,
            citationCount: buildCitations(src.pack, ctx.deps.redactor).length,
            elapsedMs: src.elapsedMs,
            startedAt,
            finishedAt,
        }, {
            store: ctx.deps.evidenceStore,
            env: ctx.deps.env,
            additionalSecrets: currentRedactionSecrets(ctx.deps),
            costClassResolver: resolveCostClass,
        });
        firstRunId ??= runId;
    }
    return firstRunId;
}
function persistConnectorAudit(store, connectors) {
    const sink = createSqliteAuditSink(store);
    const occurredAt = Date.now();
    for (const src of connectors) {
        for (const capsule of src.selected.capsules) {
            sink.emit({
                kind: "retrieval-performed",
                capsuleId: capsule.id,
                sourceIds: capsule.sourceIds,
                chunkIds: src.references.map((reference) => String(reference.chunkId)),
                referenceCount: src.references.length,
                noEvidence: src.references.length === 0,
                occurredAt,
            });
        }
    }
}
function buildHybridContextPack(ctx, sources, selected, limits, summary, assistant, knowledgeCitationCount) {
    const selectedConnectorCount = selected.filter((s) => s.kind === "connector").length;
    return {
        kind: "hybrid",
        folderSourceCount: sources.folderSourceCount,
        connectorSourceCount: sources.connectorSourceCount,
        folder: {
            ...summary,
            usage: {
                ...summary.usage,
                modelInputTokens: summary.usage.modelInputTokens + assistant.usage.promptTokens,
                modelOutputTokens: summary.usage.modelOutputTokens + assistant.usage.completionTokens,
            },
        },
        knowledge: knowledgeSummary(ctx.chat, sources.connectors, knowledgeCitationCount, selectedConnectorCount, limits.hybridMaxCandidates),
    };
}
function assembleHybridAnswer(ctx, sources, store, selected, limits, assistant, ids) {
    const { redactor } = ctx.deps;
    const citations = selectedFolderCitations(selected, redactor);
    const knowledgeCitations = selectedConnectorCitations(selected);
    const evidenceRunId = persistFolderEvidence(ctx, sources.folders);
    persistConnectorAudit(store, sources.connectors);
    const elapsedMs = sources.folders.reduce((acc, src) => acc + src.elapsedMs, 0);
    const summary = folderSummary(sources.folders, redactor);
    return {
        groundingKind: "hybrid",
        ...ids,
        evidenceRunId,
        content: redactString(redactor, assistant.content),
        citations,
        knowledgeCitations,
        uncertainty: [
            ...folderUncertainty(sources.folders, redactor),
            ...skippedUncertainty(sources.skipped, redactor),
        ],
        omittedCount: sources.folders.reduce((acc, src) => acc + src.pack.omitted.length, 0),
        elapsedMs,
        contextPack: buildHybridContextPack(ctx, sources, selected, limits, summary, assistant, knowledgeCitations.length),
    };
}
function resolveHybridAnswerer(ctx) {
    if (ctx.answer !== undefined)
        return { answer: ctx.answer };
    const model = ctx.deps.modelPortFactory(ctx.modelId);
    if (model === undefined) {
        return { status: 400, body: errorBody("NO_MODEL", "No model provider is configured.") };
    }
    return { answer: createHybridAnswerer(model, ctx.modelId, ctx.signal) };
}
export async function runHybridGroundedAsk(ctx) {
    const env = openStoreForDeps(ctx.deps);
    try {
        return await runHybridWithStore(ctx, env.store);
    }
    catch (error) {
        return mapHybridError(error, ctx.deps);
    }
    finally {
        env.close();
    }
}
async function runHybridWithStore(ctx, store) {
    const folderScopes = buildConnectedScopes(ctx.chat);
    const connectorScopes = buildLocalKnowledgeScopes(ctx.chat);
    const resolved = resolveConnectorScopes(connectorScopes, store);
    if ("status" in resolved)
        return resolved;
    const query = buildQuery(ctx.content, () => Date.now());
    const folders = await retrieveFolderPacks(ctx, folderScopes, query, ctx.folderRetriever ?? defaultRetriever(ctx.signal));
    if (isRouteResult(folders))
        return folders;
    const connectorResult = await retrieveConnectors(ctx, store, connectorScopes, resolved);
    if ("status" in connectorResult)
        return connectorResult;
    return await answerAndAssemble(ctx, store, folders, {
        folderScopeCount: folderScopes.length,
        connectorScopeCount: connectorScopes.length,
        connectorResult,
    });
}
function isRouteResult(value) {
    return !Array.isArray(value);
}
async function answerAndAssemble(ctx, store, folders, meta) {
    const answerer = resolveHybridAnswerer(ctx);
    if ("status" in answerer)
        return answerer;
    const limits = currentGroundingLimits(ctx.deps);
    const selected = buildUnifiedSelection(ctx, folders, meta.connectorResult.retrieved, store);
    const user = buildRerankedHybridUserMessage(ctx.content, selected, ctx.deps.redactor);
    const assistant = normalizeGroundedAnswerPayload(await answerer.answer(HYBRID_SYSTEM_PROMPT, user));
    const [userMessage, assistantMessage] = persistGroundedExchange(ctx.deps, ctx.chat.id, redactString(ctx.deps.redactor, ctx.content), redactString(ctx.deps.redactor, assistant.content));
    const answer = assembleHybridAnswer(ctx, {
        folders,
        connectors: meta.connectorResult.retrieved,
        skipped: meta.connectorResult.skipped,
        folderSourceCount: meta.folderScopeCount,
        connectorSourceCount: meta.connectorScopeCount,
    }, store, selected, limits, assistant, { userMessageId: userMessage.id, assistantMessageId: assistantMessage.id });
    return { status: 200, body: answer };
}
// Issue #154 (GAP-B) — a GatewayError is redacted inside mappedGatewayError (shared with the
// single-source path). The non-gateway `Error` fallback carries an arbitrary dynamic message that
// can echo a provider endpoint or token, so it is scrubbed through the SAME boundary before it
// reaches the wire.
function mapHybridError(error, deps) {
    const gatewayResult = mappedGatewayError(error, deps);
    if (gatewayResult !== undefined)
        return gatewayResult;
    if (error instanceof Error) {
        return internalError(redact(error.message, currentRedactionSecrets(deps)));
    }
    throw error;
}
