import * as fs from 'node:fs';
import * as path from 'node:path';
import { calculateDeterministicHash } from '../artifacts/hash.js';
import { createChunker } from '../chunkers/chunker.js';
import { requireEmbedding } from '../config-schema.js';
import { createEmbedder } from '../embedders/embedder.js';
import { TotemDatabaseError } from '../errors.js';
import { withLock } from '../lock.js';
import { sanitizeForIngestion } from '../sanitize.js';
import { LanceStore } from '../store/lance-store.js';
import { sanitizeForTerminal } from '../terminal-sanitize.js';
import { getChangedFiles, getHeadSha, resolveFiles } from './file-resolver.js';
const EMBED_BATCH_SIZE = 100;
const SYNC_STATE_FILE = 'cache/sync-state.json';
const INDEX_META_FILE = 'cache/index-meta.json';
function readIndexMeta(totemDir) {
    const metaPath = path.join(totemDir, INDEX_META_FILE);
    try {
        const parsed = JSON.parse(fs.readFileSync(metaPath, 'utf-8'));
        if (parsed &&
            typeof parsed.provider === 'string' &&
            typeof parsed.model === 'string' &&
            typeof parsed.dimensions === 'number' &&
            typeof parsed.lastSync === 'string') {
            return parsed;
        }
        return null;
    }
    catch {
        return null;
    }
}
function writeIndexMeta(totemDir, meta) {
    const metaPath = path.join(totemDir, INDEX_META_FILE);
    const dir = path.dirname(metaPath);
    fs.mkdirSync(dir, { recursive: true });
    fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2) + '\n', 'utf-8');
}
/**
 * Verify that the current embedding config matches the index.
 * Throws TotemDatabaseError if there's a mismatch.
 */
/**
 * Verify that the current embedding config matches the index.
 * Compares provider name only — dimensions are checked at sync time
 * when the actual embedder resolves its effective dimensions.
 * Throws TotemDatabaseError if the provider has changed.
 */
export function verifyIndexMeta(totemDir, config) {
    const embedding = config.embedding;
    if (!embedding)
        return; // Lite tier — no index to verify
    const meta = readIndexMeta(totemDir);
    if (!meta)
        return; // No meta yet — first sync hasn't happened
    if (meta.provider !== embedding.provider) {
        throw new TotemDatabaseError(`Index was built with ${meta.provider} (${meta.dimensions}d) but config now uses ${embedding.provider}.`, "Run 'totem sync --full' to rebuild the index.", 'DATABASE_MISMATCH');
    }
    // If explicit dimensions are set and don't match, warn
    if (embedding.dimensions && meta.dimensions !== embedding.dimensions) {
        throw new TotemDatabaseError(`Index was built with ${meta.dimensions}d vectors but config now specifies ${embedding.dimensions}d.`, "Run 'totem sync --full' to rebuild the index.", 'DATABASE_MISMATCH');
    }
}
function readSyncState(totemDir, onProgress) {
    const statePath = path.join(totemDir, SYNC_STATE_FILE);
    try {
        const raw = fs.readFileSync(statePath, 'utf-8');
        const parsed = JSON.parse(raw);
        if (typeof parsed.lastSyncSha === 'string' &&
            parsed.lastSyncSha &&
            typeof parsed.timestamp === 'number') {
            return parsed;
        }
        onProgress?.(`Warning: Ignoring malformed sync state file at ${statePath}.`);
        return null;
    }
    catch (err) {
        // ENOENT is expected on first run — don't warn
        if (err instanceof Error && err.code === 'ENOENT') {
            return null;
        }
        onProgress?.(`Warning: Failed to read sync state: ${err instanceof Error ? err.message : String(err)}`);
        return null;
    }
}
function writeSyncState(totemDir, state) {
    const statePath = path.join(totemDir, SYNC_STATE_FILE);
    const dir = path.dirname(statePath);
    fs.mkdirSync(dir, { recursive: true });
    fs.writeFileSync(statePath, JSON.stringify(state, null, 2) + '\n', 'utf-8');
}
/** Current schema identifier persisted to `.totem/index-manifest.json`. */
export const INDEX_MANIFEST_SCHEMA = 'totem-index-manifest-v0.2';
/**
 * Builds an {@link IndexManifest} payload from the per-document data and the
 * sync run's HEAD SHA + write timestamp.
 *
 * - `writtenAt` is serialized to ISO-8601.
 * - `gitCommit` is included as `git:<sha>` only when `input.headSha` is a
 *   non-empty string; null / undefined / `''` cause the field to be omitted
 *   entirely (no `sha1:unknown` or other synthesized fake-presence values).
 * - `schema` is always set to {@link INDEX_MANIFEST_SCHEMA}.
 *
 * Pure helper — does not touch the filesystem. The sync pipeline is
 * responsible for atomic write/rename of the returned payload.
 */
export function buildIndexManifest(input) {
    const manifest = {
        schema: INDEX_MANIFEST_SCHEMA,
        writtenAt: input.writtenAt.toISOString(),
        documents: input.documents,
    };
    if (input.headSha) {
        manifest.gitCommit = `git:${input.headSha}`;
    }
    return manifest;
}
export async function runSync(config, options) {
    const { projectRoot, onProgress } = options;
    const log = onProgress ?? (() => { });
    const totemDir = path.join(projectRoot, config.totemDir);
    return withLock(totemDir, () => runSyncInner(config, options), (msg) => log(msg));
}
/**
 * Normalize a path separator to forward slashes for orphan COMPARISON only.
 * Deletes always use the raw stored path (mmnto-ai/totem#2151 W1): `deleteByFile`
 * matches the stored literal, so a legacy `src\x.ts` row must delete with
 * backslashes while never being false-orphaned against a forward-slash live path.
 */
function normalizeRel(p) {
    return p.replace(/\\/g, '/');
}
/**
 * Reconcile indexed paths against the working tree: an indexed path with no live
 * counterpart is an orphan to purge (mmnto-ai/totem#2151). Compares on a
 * separator-normalized key but RETURNS THE RAW indexed path (the caller deletes
 * via `deleteByFile`, a stored-literal match). Independent of the git diff
 * window — that independence is what self-heals deletions the baseline already
 * advanced past, and de-targeted / newly-ignored files git never reports.
 */
export function computeOrphanPaths(indexedRawPaths, liveRelativePaths) {
    const live = new Set(liveRelativePaths.map(normalizeRel));
    return indexedRawPaths.filter((raw) => !live.has(normalizeRel(raw)));
}
/**
 * Order-normalized content hash of the effective index-exclusion set
 * (`ignorePatterns` ∪ `indexIgnorePatterns`). Patterns are sorted before hashing
 * so reordering the config is not a spurious mismatch — only a membership change
 * (a pattern added or removed) moves the hash. Persisted in sync state to detect
 * ignore-pattern REMOVALS between incremental syncs (mmnto-ai/totem#2366), which
 * the git-diff window alone never surfaces. Uses the one deterministic-hash
 * convention shared with run artifacts.
 */
export function hashIndexExclusionSet(patterns) {
    // calculateDeterministicHash treats array order as significant — the .sort()
    // here IS the order-independence invariant. Do not remove or move it.
    return calculateDeterministicHash([...patterns].sort());
}
/**
 * Files that are index-eligible now but absent from the index — the symmetric
 * complement of {@link computeOrphanPaths} (mmnto-ai/totem#2366). Removing an
 * ignore pattern makes files eligible without changing their bytes, so the git
 * diff window never reports them; this recovers them by membership alone
 * (live-vs-indexed), independent of the diff. Compares on a separator-normalized
 * key like the orphan pass but RETURNS THE RAW live relative path (the caller
 * re-embeds via the resolved file record).
 */
export function computeNewlyEligiblePaths(liveRelativePaths, indexedRawPaths) {
    const indexed = new Set(indexedRawPaths.map(normalizeRel));
    return liveRelativePaths.filter((rel) => !indexed.has(normalizeRel(rel)));
}
async function runSyncInner(config, options) {
    const { projectRoot, onProgress } = options;
    let incremental = options.incremental;
    const log = onProgress ?? (() => { });
    // 0. Capture HEAD SHA early — before any async work that might race with new commits
    const headSha = getHeadSha(projectRoot, log);
    // 1. Create embedder
    const embedding = requireEmbedding(config);
    log('Initializing embedding provider...');
    const embedder = createEmbedder(embedding);
    // 2. Connect to store
    const storePath = path.join(projectRoot, config.lanceDir);
    const store = new LanceStore(storePath, embedder, { absolutePathRoot: projectRoot });
    await store.connect();
    // 2b. Auto-heal: force full sync when incremental is requested but DB is empty
    if (incremental && (await store.isEmpty())) {
        log('Empty database detected. Forcing full sync...');
        incremental = false;
    }
    // 3. Resolve files to process. Index exclusion unions both keys:
    // `ignorePatterns` (legacy dual-scope) + `indexIgnorePatterns` (index-only,
    // mmnto-ai/totem#1748) — only the former also gates lint/shield scope.
    const totemDir = path.join(projectRoot, config.totemDir);
    const exclusionPatterns = [...config.ignorePatterns, ...config.indexIgnorePatterns];
    const allFiles = resolveFiles(config.targets, projectRoot, exclusionPatterns, log);
    // Order-normalized hash of the effective exclusion set — persisted below and
    // compared on the next incremental sync to catch pattern REMOVALS (#2366).
    const indexExclusionHash = hashIndexExclusionSet(exclusionPatterns);
    let filesToProcess;
    let deletedPaths = [];
    if (incremental) {
        // Persisted sync state carries both the diff baseline (lastSyncSha) and the
        // exclusion-set hash used for removal reconciliation (mmnto-ai/totem#2366).
        const syncState = readSyncState(totemDir, log);
        // Determine the ref to diff against: saved sync state > fallback HEAD~1
        let sinceRef = 'HEAD~1';
        if (!options.changedFiles && syncState) {
            sinceRef = syncState.lastSyncSha;
            log(`Resuming from last sync at ${sinceRef.slice(0, 8)}...`);
        }
        const changedPaths = options.changedFiles ?? getChangedFiles(projectRoot, sinceRef, log);
        if (changedPaths === null) {
            log('Git diff failed, falling back to full sync...');
            await store.reset();
            filesToProcess = allFiles;
            log(`Full sync (fallback): ${filesToProcess.length} files to process`);
        }
        else {
            const changedSet = new Set(changedPaths);
            let newlyEligible = new Set();
            // Reconciliation is NOT diff-derived (mmnto-ai/totem#2151, #2366): both the
            // orphan purge and the newly-eligible enqueue below read the indexed set
            // once and compare against the live tree, independent of the git diff
            // window. On a getDistinctPaths read failure, skip BOTH this run (warn, no
            // false purge / no spurious enqueue); a later sync reconciles.
            try {
                const indexedPaths = await store.getDistinctPaths();
                const liveRelPaths = allFiles.map((f) => f.relativePath);
                // Deletion (ADDITION direction): purge any indexed path no longer in the
                // working tree — self-heals deletions, renames-into-ignored, and
                // de-targeted / newly-ignored files even after the baseline advanced past
                // them.
                deletedPaths = computeOrphanPaths(indexedPaths, liveRelPaths);
                // Ignore-pattern REMOVAL reconciliation: symmetric to the orphan pass. A
                // pattern removed from the exclusion set makes files index-eligible
                // without changing their bytes, so the diff window never surfaces them. A
                // change in the stored order-normalized exclusion hash — or its ABSENCE
                // (state predating #2366) — enqueues the newly-eligible set (live −
                // indexed). The union naturally reduces to the diff alone when the
                // exclusion set is unchanged, and on a pure ADDITION the newly-eligible
                // files are all already in the changed set.
                if (syncState?.indexExclusionHash !== indexExclusionHash) {
                    newlyEligible = new Set(computeNewlyEligiblePaths(liveRelPaths, indexedPaths));
                }
                // totem-context: intentional warn+skip on a getDistinctPaths read failure (mmnto-ai/totem#2151 Q3, cohort-endorsed) — never a false purge; a later sync reconciles, and the Warning below means it is not silent degradation.
            }
            catch (err) {
                log(`  Warning: orphan reconciliation skipped (getDistinctPaths failed): ${err instanceof Error ? err.message : String(err)}`);
            }
            // Re-embed the changed files (diff-scoped) UNION the newly-eligible set an
            // ignore-pattern removal exposed (#2366); every path in either set still
            // exists (both derive from allFiles / the live tree).
            filesToProcess = allFiles.filter((f) => changedSet.has(f.relativePath) || newlyEligible.has(f.relativePath));
            // Count only the extras the exclusion-set change pulled back in (those not
            // already surfaced by the diff), so the log reports honest reconciliation work.
            const reconciledCount = [...newlyEligible].filter((rel) => !changedSet.has(rel)).length;
            log(`Incremental sync: ${filesToProcess.length} file(s) to process` +
                (reconciledCount > 0 ? `, ${reconciledCount} newly-eligible` : '') +
                (deletedPaths.length > 0 ? `, ${deletedPaths.length} orphan candidate(s)` : '') +
                ` (of ${allFiles.length} total)`);
        }
    }
    else {
        log('Full sync: resetting index...');
        await store.reset();
        filesToProcess = allFiles;
        log(`Full sync: ${filesToProcess.length} files to process`);
    }
    // 3b. Purge orphaned chunks (reconciliation result) before ingesting new ones.
    let orphansPurged = 0;
    for (const deletedPath of deletedPaths) {
        try {
            await store.deleteByFile(deletedPath);
            orphansPurged++;
            log(`  Purged chunks for orphaned file: ${deletedPath}`);
            // totem-context: best-effort per-file purge — warn+skip; the orphan stays
            // indexed and the next reconciliation retries it (deleteByFile is idempotent),
            // so one bad row never aborts the whole sync (mmnto-ai/totem#2151).
        }
        catch (err) {
            log(`  Warning: failed to purge ${deletedPath}: ${err instanceof Error ? err.message : String(err)}`);
        }
    }
    if (orphansPurged > 0) {
        log(`Purged ${orphansPurged} orphaned file(s) from the index.`);
    }
    // 4. Chunk files and stream to LanceDB in batches (bounded memory)
    let totalChunks = 0;
    let buffer = [];
    async function flushBuffer() {
        if (buffer.length === 0)
            return;
        await store.insert(buffer);
        totalChunks += buffer.length;
        log(`  Embedded ${totalChunks} chunks so far`);
        buffer = [];
    }
    for (const file of filesToProcess) {
        const safeRel = sanitizeForTerminal(file.relativePath);
        log(`Chunking ${safeRel}...`);
        let content;
        try {
            // Symlink guard at the READ site (mmnto-ai/totem#2354, #2356 review):
            // resolveFiles skips symlinks at discovery, but this read happens later,
            // so a path swapped to a symlink after resolution would still be followed
            // by readFileSync. Re-check here and skip — closes the discovery→read
            // TOCTOU gap.
            if (fs.lstatSync(file.absolutePath).isSymbolicLink()) {
                log(`  Skipping symlink (not indexed): ${safeRel}`);
                continue;
            }
            content = fs.readFileSync(file.absolutePath, 'utf-8');
            // totem-context: intentional skip — a per-file read failure (vanished/raced file, permission, decode) logs and skips just that file so sync continues over the rest; aborting the whole sync over one unreadable file would be the drift, not the skip.
        }
        catch (err) {
            log(`  Skipping (read error: ${sanitizeForTerminal(err instanceof Error ? err.message : String(err))}): ${safeRel}`);
            continue;
        }
        const chunker = createChunker(file.target.strategy);
        const chunks = chunker.chunk(content, file.relativePath, file.target.type);
        if (chunks.length === 0) {
            log(`  No chunks extracted from ${file.relativePath}`);
            continue;
        }
        // For incremental: delete old chunks for this file before inserting new ones
        if (incremental) {
            await store.deleteByFile(file.relativePath);
        }
        // Sanitize chunk content before embedding (adversarial ingestion scrubbing)
        // Deduplicate warnings per file to avoid log spam on files with widespread issues
        const warnedMessages = new Set();
        const dedupeWarn = (msg) => {
            if (!warnedMessages.has(msg)) {
                warnedMessages.add(msg);
                log(msg);
            }
        };
        for (const chunk of chunks) {
            const sanitizeOpts = {
                chunkType: chunk.type,
                filePath: file.relativePath,
                onWarn: dedupeWarn,
            };
            chunk.content = sanitizeForIngestion(chunk.content, sanitizeOpts);
            chunk.contextPrefix = sanitizeForIngestion(chunk.contextPrefix, sanitizeOpts);
        }
        buffer.push(...chunks);
        log(`  ${chunks.length} chunks from ${file.relativePath}`);
        // Flush when buffer reaches batch size to keep memory bounded
        if (buffer.length >= EMBED_BATCH_SIZE) {
            await flushBuffer();
        }
    }
    // Flush remaining chunks
    await flushBuffer();
    log(`Sync complete: ${totalChunks} chunks from ${filesToProcess.length} files`);
    // Build/rebuild FTS index for hybrid search (FTS indexes don't auto-update on
    // add — or on delete: a purge-only sync must still rebuild so the FTS view
    // drops the orphaned files' content, mmnto-ai/totem#2151 Tenet 20).
    // Sum (not ||): both are non-negative counts, so > 0 iff either is — and it
    // sidesteps the logical-OR numeric-default lint without changing the meaning.
    if (totalChunks + orphansPurged > 0) {
        log('Building FTS index for hybrid search...');
        await store.createFtsIndex();
        log(store.ftsIndexReady
            ? 'FTS index ready.'
            : 'FTS index skipped (hybrid search will use vector-only).');
    }
    // Persist sync state so the next incremental sync knows where to diff from and
    // can detect an ignore-pattern change (including a removal) via the hash (#2366).
    if (headSha) {
        writeSyncState(totemDir, { lastSyncSha: headSha, timestamp: Date.now(), indexExclusionHash });
    }
    // Persist index metadata for dimension mismatch detection
    writeIndexMeta(totemDir, {
        provider: embedding.provider,
        model: embedding.model ?? 'default',
        dimensions: embedder.dimensions,
        lastSync: new Date().toISOString(),
    });
    // Persist index manifest for downstream consumers (e.g. totem-status Visor)
    try {
        // Single timestamp threaded through both manifestDocuments() and buildIndexManifest
        // so documents[].lastSynced == manifest.writtenAt, per the v0.2 contract.
        const writtenAt = new Date();
        const docs = await store.manifestDocuments(writtenAt);
        const manifest = buildIndexManifest({ documents: docs, headSha, writtenAt });
        const manifestPath = path.join(totemDir, 'index-manifest.json');
        const tmpManifestPath = manifestPath + '.tmp';
        fs.writeFileSync(tmpManifestPath, JSON.stringify(manifest, null, 2) + '\n', 'utf-8');
        fs.renameSync(tmpManifestPath, manifestPath);
    }
    catch (err) {
        const detail = err instanceof Error ? err.message : typeof err === 'string' ? err : 'unknown error';
        log(`Warning: Failed to write index-manifest.json: ${detail}`);
    }
    // Get total chunk count from the store (includes pre-existing chunks from incremental syncs)
    let totalStoredChunks = totalChunks;
    try {
        totalStoredChunks = await store.count();
    }
    catch {
        // Count failure should not break sync — fall back to chunks processed this run
    }
    return {
        chunksProcessed: totalChunks,
        filesProcessed: filesToProcess.length,
        totalChunks: totalStoredChunks,
        orphansPurged,
    };
}
//# sourceMappingURL=pipeline.js.map