import { readFile, stat, unlink, rm, mkdir as mkdirAsync } from "node:fs/promises";
import { join, relative, resolve, basename, dirname } from "node:path";
import { openSync, closeSync, statSync, unlinkSync, writeFileSync, renameSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
import { EXTRACTOR_VERSIONS } from "./project-tools.js";
import { extractMarkdownSymbols, extractPrismaSymbols, extractAstroSymbols, extractConversationSymbols } from "../parser/symbol-extractor.js";
import { runTreeSitterParse } from "../parser/parser-pool.js";
import { extractSqlSymbols, stripJinjaTokens } from "../parser/extractors/sql.js";
import { getLanguageForExtension, getLanguageForPath } from "../parser/parser-manager.js";
import { saveIndex, loadIndex, loadIndexOrStale, getIndexPath, saveIncremental, removeFileFromIndex } from "../storage/index-store.js";
import { clearTsconfigCache } from "../utils/tsconfig-paths.js";
import { registerRepo, listRepos as listRegistryRepos, getRepo, removeRepo, getRepoName, updateRepoMeta, resolveRegisteredRepoMeta } from "../storage/registry.js";
import { startWatcher, stopWatcher } from "../storage/watcher.js";
import { buildBM25Index } from "../search/bm25.js";
import { buildSymbolText, createEmbeddingProvider } from "../search/semantic.js";
import { loadEmbeddings, saveEmbeddings, saveEmbeddingMeta, getEmbeddingPath, getEmbeddingMetaPath, batchEmbed } from "../storage/embedding-store.js";
import { saveChunks, saveChunkEmbeddings, loadChunkEmbeddings, getChunkPath, getChunkEmbeddingPath } from "../storage/chunk-store.js";
import { chunkFile, chunkBySymbols } from "../search/chunker.js";
import { loadConfig } from "../config.js";
import { validateGitUrl, validateGitRef } from "../utils/git-validation.js";
import { walkDirectory } from "../utils/walk.js";
import { onFileChanged as scanOnChanged, onFileDeleted as scanOnDeleted, scanFileForSecrets } from "./secret-tools.js";
import { getGraphPath } from "../storage/graph-store.js";
const PARSE_CONCURRENCY = 8;
const CHUNK_EMBEDDING_BATCH_SIZE = 96;
const GIT_CLONE_TIMEOUT_MS = 120_000;
const GIT_CHECKOUT_TIMEOUT_MS = 30_000;
const GIT_PULL_TIMEOUT_MS = 60_000;
/**
 * Default cap on files returned by walkDirectory during indexFolder. Without
 * this, repos with hundreds of thousands of code files (large CMS, monorepos,
 * vendored data sets) accumulate symbols in memory and OOM the Node process,
 * which manifests to MCP clients as a "Connection closed" error mid-index.
 *
 * 50_000 covers every project we've seen in practice; the largest indexed
 * monorepo (tgm-survey-platform) is around 5_000 files. Override via the
 * MCP `max_files` argument or CODESIFT_MAX_FILES env var when a legitimately
 * larger repo needs full coverage.
 */
const DEFAULT_MAX_FILES = 50_000;
function getDefaultMaxFiles() {
    const envVal = process.env.CODESIFT_MAX_FILES;
    if (envVal) {
        const parsed = Number.parseInt(envVal, 10);
        if (Number.isFinite(parsed) && parsed > 0)
            return parsed;
    }
    return DEFAULT_MAX_FILES;
}
// Active watchers and in-memory indexes keyed by repo name
const activeWatchers = new Map();
const bm25Indexes = new Map();
const codeIndexes = new Map();
const embeddingCaches = new Map();
// Tracks last successful full indexFolder run keyed by absolute rootPath, used
// to short-circuit redundant scans while a watcher is keeping the index live.
// Populated at the end of every indexFolder call that completes normally.
const lastFullIndexAt = new Map();
/**
 * Window during which a re-run of indexFolder against the same root is treated
 * as redundant when a watcher is active. Telemetry showed 35 calls > 30s and a
 * 786s outlier — these are agents defensively re-indexing repos the watcher is
 * already maintaining. 60s matches the "fresh enough" threshold used elsewhere.
 */
const INDEX_FOLDER_REDUNDANT_WINDOW_MS = 60_000;
/** Test-only — clear short-circuit state between cases. */
export function resetIndexFolderRedundancyForTesting() {
    lastFullIndexAt.clear();
}
/**
 * Parse a single file and extract its symbols + metadata.
 * Returns null if the file cannot be parsed.
 */
async function parseOneFile(filePath, repoRoot, repoName) {
    try {
        const stat = await import("node:fs/promises").then((fs) => fs.stat(filePath));
        const source = await readFile(filePath, "utf-8");
        const relPath = relative(repoRoot, filePath);
        const baseName = filePath.split("/").pop() ?? "";
        // Use full-path resolver so multi-dot suffixes like `.gradle.kts` beat
        // single-extension lookups (which would otherwise map to plain Kotlin).
        const language = getLanguageForPath(filePath)
            ?? (baseName.startsWith(".env") ? "config" : "unknown");
        let symbols;
        let effectiveLanguage = language;
        if (language === "markdown") {
            symbols = extractMarkdownSymbols(source, relPath, repoName);
        }
        else if (language === "prisma") {
            symbols = extractPrismaSymbols(source, relPath, repoName);
        }
        else if (language === "astro") {
            symbols = extractAstroSymbols(source, relPath, repoName);
        }
        else if (language === "conversation") {
            symbols = extractConversationSymbols(source, relPath, repoName);
        }
        else if (language === "sql") {
            // SQL: regex extractor, no tree-sitter. Detect Jinja/dbt templates.
            const hasJinja = /\{\{|\{%|\{#/.test(source);
            if (hasJinja) {
                const stripped = stripJinjaTokens(source);
                symbols = extractSqlSymbols(stripped, relPath, repoName, source);
                effectiveLanguage = "sql-jinja";
            }
            else {
                symbols = extractSqlSymbols(source, relPath, repoName);
            }
        }
        else if (language === "config" || language === "text_stub") {
            // text_stub: Swift/Dart/Scala/etc. — indexed as FileEntry but no symbol
            // extraction until a tree-sitter grammar + extractor is added.
            // search_text (ripgrep path) and scan_secrets still work on these files.
            symbols = [];
        }
        else {
            // Tree-sitter languages (TS/JS/Python/Go/Rust/Java/Ruby/PHP/CSS/Kotlin):
            // dispatch to the worker pool. Synchronously-hung WASM parses kill the
            // worker (terminated on timeout) instead of the MCP server itself.
            // See src/parser/parser-pool.ts for details.
            try {
                symbols = await runTreeSitterParse({ filePath, source, language, relPath, repoName });
            }
            catch (err) {
                const message = err instanceof Error ? err.message : String(err);
                console.warn(`[codesift] worker parse failed for ${relPath}: ${message}`);
                return null;
            }
        }
        const entry = {
            path: relPath,
            language: effectiveLanguage,
            symbol_count: symbols.length,
            last_modified: Date.now(),
            mtime_ms: Math.round(stat.mtimeMs),
        };
        return { symbols, entry };
    }
    catch (err) {
        const message = err instanceof Error ? err.message : String(err);
        console.warn(`[codesift] Failed to parse ${relative(repoRoot, filePath)}: ${message}`);
        return null;
    }
}
/**
 * Parse files in parallel batches.
 */
async function parseFiles(files, repoRoot, repoName) {
    const allSymbols = [];
    const fileEntries = [];
    for (let i = 0; i < files.length; i += PARSE_CONCURRENCY) {
        const batch = files.slice(i, i + PARSE_CONCURRENCY);
        const results = await Promise.all(batch.map((filePath) => parseOneFile(filePath, repoRoot, repoName)));
        for (const result of results) {
            if (result) {
                allSymbols.push(...result.symbols);
                fileEntries.push(result.entry);
            }
        }
    }
    return { symbols: allSymbols, fileEntries };
}
// ---------------------------------------------------------------------------
// Dirty propagation — mark caller files stale when a callee signature changes
// ---------------------------------------------------------------------------
/**
 * Compute a hash of a symbol's public interface (name + kind + signature).
 * Body changes don't trigger propagation — only signature changes.
 */
function computeSignatureHash(sym) {
    const key = `${sym.name}|${sym.kind}|${sym.signature ?? ""}`;
    return createHash("sha256").update(key).digest("hex").slice(0, 16);
}
/**
 * Detect signature changes and mark caller files as stale.
 * Returns the set of files marked stale.
 */
function propagateDirtySignatures(oldSymbols, newSymbols, fileEntries) {
    // Build old signature hashes
    const oldHashes = new Map();
    for (const sym of oldSymbols) {
        oldHashes.set(sym.id, computeSignatureHash(sym));
    }
    // Find symbols with changed signatures
    const changedSymbolFiles = new Set();
    for (const sym of newSymbols) {
        const oldHash = oldHashes.get(sym.id);
        if (oldHash && oldHash !== computeSignatureHash(sym)) {
            changedSymbolFiles.add(sym.file);
        }
    }
    if (changedSymbolFiles.size === 0)
        return new Set();
    // Find files that import from changed files (1 level of callers)
    // Use a simple heuristic: check if any symbol source mentions a changed file's name
    const changedBasenames = new Set();
    for (const f of changedSymbolFiles) {
        const base = f.split("/").pop()?.replace(/\.\w+$/, "");
        if (base)
            changedBasenames.add(base);
    }
    const staleFiles = new Set();
    for (const sym of newSymbols) {
        if (changedSymbolFiles.has(sym.file))
            continue; // Don't mark the changed file itself
        if (!sym.source)
            continue;
        for (const base of changedBasenames) {
            if (sym.source.includes(base)) {
                staleFiles.add(sym.file);
                break;
            }
        }
    }
    // Mark stale in file entries (clear mtime so next index re-parses them)
    for (const entry of fileEntries) {
        if (staleFiles.has(entry.path)) {
            entry.stale = true;
            delete entry.mtime_ms; // Force re-parse on next indexFolder
        }
    }
    return staleFiles;
}
/**
 * Embed symbols using the configured embedding provider.
 * Non-fatal — BM25 search still works if embedding fails.
 */
export async function embedSymbols(symbols, indexPath, repoName, config) {
    if (!config.embeddingProvider)
        return;
    const embeddingPath = getEmbeddingPath(indexPath);
    const metaPath = getEmbeddingMetaPath(indexPath);
    try {
        const provider = createEmbeddingProvider(config.embeddingProvider, config);
        const symbolTexts = new Map(symbols.map((s) => [s.id, buildSymbolText(s)]));
        const existing = await loadEmbeddings(embeddingPath);
        const embeddings = await batchEmbed(symbolTexts, existing, (texts) => provider.embed(texts, "document"), config.embeddingBatchSize, repoName);
        await saveEmbeddings(embeddingPath, embeddings);
        await saveEmbeddingMeta(metaPath, {
            model: provider.model,
            provider: config.embeddingProvider,
            dimensions: provider.dimensions,
            symbol_count: embeddings.size,
            updated_at: Date.now(),
        });
        embeddingCaches.set(repoName, embeddings);
    }
    catch (err) {
        const message = err instanceof Error ? err.message : String(err);
        console.error(`[codesift] Embedding failed for ${repoName}: ${message}`);
    }
}
/**
 * Read files in parallel batches and split each into chunks.
 */
async function readAndChunkFiles(fileEntries, rootPath, repoName, symbols) {
    const allChunks = [];
    for (let i = 0; i < fileEntries.length; i += PARSE_CONCURRENCY) {
        const batch = fileEntries.slice(i, i + PARSE_CONCURRENCY);
        const batchResults = await Promise.all(batch.map(async (entry) => {
            const fullPath = join(rootPath, entry.path);
            try {
                const content = await readFile(fullPath, "utf-8");
                if (symbols) {
                    const fileSymbols = symbols
                        .filter((s) => s.file === entry.path)
                        .map((s) => ({ name: s.name, start_line: s.start_line, end_line: s.end_line }));
                    return chunkBySymbols(entry.path, content, repoName, fileSymbols);
                }
                return chunkFile(entry.path, content, repoName);
            }
            catch (err) {
                const message = err instanceof Error ? err.message : String(err);
                console.warn(`[codesift] Failed to read ${entry.path} for chunking: ${message}`);
                return [];
            }
        }));
        for (const chunks of batchResults) {
            allChunks.push(...chunks);
        }
    }
    return allChunks;
}
/**
 * Embed file chunks using the configured embedding provider.
 * Non-fatal — symbol-level and BM25 search still work if this fails.
 */
async function embedChunks(fileEntries, rootPath, repoName, indexPath, config, symbols) {
    if (!config.embeddingProvider)
        return;
    const chunkPath = getChunkPath(indexPath);
    const chunkEmbeddingPath = getChunkEmbeddingPath(indexPath);
    try {
        const provider = createEmbeddingProvider(config.embeddingProvider, config);
        const existingChunkEmbeddings = await loadChunkEmbeddings(chunkEmbeddingPath) ?? new Map();
        const allChunks = await readAndChunkFiles(fileEntries, rootPath, repoName, symbols);
        if (allChunks.length > 0) {
            const chunkTexts = new Map(allChunks.map((c) => [c.id, c.text]));
            const chunkEmbeddings = await batchEmbed(chunkTexts, existingChunkEmbeddings, (texts) => provider.embed(texts, "document"), CHUNK_EMBEDDING_BATCH_SIZE, `${repoName}:chunks`);
            await saveChunks(chunkPath, allChunks);
            await saveChunkEmbeddings(chunkEmbeddingPath, chunkEmbeddings);
        }
    }
    catch (err) {
        const message = err instanceof Error ? err.message : String(err);
        console.error(`[codesift] Chunk embedding failed for ${repoName}: ${message}`);
    }
}
export async function indexFolder(folderPath, options) {
    if (!folderPath || typeof folderPath !== "string") {
        throw new Error("folderPath is required and must be a non-empty string");
    }
    const rootPath = resolve(folderPath);
    const repoName = getRepoName(rootPath);
    // Short-circuit: if a watcher is already keeping the index for this root
    // live and we re-indexed recently, return a skipped status instead of
    // walking the filesystem again.
    if (!options?.force) {
        const lastTs = lastFullIndexAt.get(rootPath);
        const watcher = activeWatchers.get(repoName);
        if (watcher && lastTs && Date.now() - lastTs < INDEX_FOLDER_REDUNDANT_WINDOW_MS) {
            return {
                repo: repoName,
                root: rootPath,
                file_count: 0,
                symbol_count: 0,
                duration_ms: 0,
                status: "skipped",
                reason: "watcher active, recent index",
                last_indexed: new Date(lastTs).toISOString(),
                hint: "pass force=true to override",
            };
        }
    }
    // Clear tsconfig path resolver cache so config edits between runs take effect.
    // The two-level cache (configCache + dirToConfigCache) is module-level and
    // would otherwise serve stale alias mappings if a user edited tsconfig.json
    // between successive index_folder calls within the same MCP server process.
    clearTsconfigCache();
    const config = loadConfig();
    const startTime = Date.now();
    const indexPath = getIndexPath(config.dataDir, rootPath);
    // Read .codesiftignore for user-defined exclude patterns
    let excludePatterns;
    try {
        const ignoreContent = await readFile(join(rootPath, ".codesiftignore"), "utf-8");
        excludePatterns = ignoreContent
            .split("\n")
            .map((line) => line.replace(/#.*$/, "").trim())
            .filter((line) => line.length > 0);
        if (excludePatterns.length === 0)
            excludePatterns = undefined;
    }
    catch {
        // .codesiftignore not found — proceed without patterns
    }
    // Walk directory and collect parseable files. maxFiles caps unbounded walks
    // (huge monorepos, vendored data sets) before they OOM the process — see
    // DEFAULT_MAX_FILES rationale above.
    const maxFiles = options?.max_files ?? getDefaultMaxFiles();
    const files = await walkDirectory(rootPath, {
        includePaths: options?.include_paths,
        excludePatterns,
        maxFiles,
        fileFilter: (ext, name) => !!getLanguageForExtension(ext) || (name?.startsWith(".env") ?? false),
    });
    const hitFileLimit = files.length >= maxFiles;
    if (hitFileLimit) {
        console.error(`[codesift] index_folder: ${rootPath} hit max_files=${maxFiles}; ` +
            `partial index. Pass include_paths to scope the walk or raise max_files.`);
    }
    // mtime-based incremental: skip files unchanged since last index
    const existing = await loadIndex(indexPath);
    const mtimeMap = new Map();
    if (existing) {
        for (const f of existing.files) {
            if (f.mtime_ms)
                mtimeMap.set(f.path, f.mtime_ms);
        }
    }
    const filesToParse = [];
    const keptSymbols = [];
    const keptEntries = [];
    if (mtimeMap.size > 0) {
        const { stat } = await import("node:fs/promises");
        for (const filePath of files) {
            const relPath = relative(rootPath, filePath);
            const prevMtime = mtimeMap.get(relPath);
            if (prevMtime !== undefined) {
                const fileEntry = existing.files.find((f) => f.path === relPath);
                // Force re-parse if file is marked stale (callee signature changed)
                if (fileEntry?.stale) {
                    filesToParse.push(filePath);
                    continue;
                }
                try {
                    const st = await stat(filePath);
                    if (Math.round(st.mtimeMs) === prevMtime) {
                        // File unchanged — keep existing symbols
                        const fileSymbols = existing.symbols.filter((s) => s.file === relPath);
                        if (fileEntry) {
                            keptSymbols.push(...fileSymbols);
                            keptEntries.push(fileEntry);
                            continue;
                        }
                    }
                }
                catch { /* file may have been deleted — reparse */ }
            }
            filesToParse.push(filePath);
        }
    }
    else {
        filesToParse.push(...files);
    }
    // Parse only changed/new files
    const { symbols: parsedSymbols, fileEntries: parsedEntries } = await parseFiles(filesToParse, rootPath, repoName);
    const symbols = [...keptSymbols, ...parsedSymbols];
    const fileEntries = [...keptEntries, ...parsedEntries];
    // Dirty propagation: detect signature changes and mark caller files stale
    if (existing && filesToParse.length > 0 && filesToParse.length < files.length) {
        const staleFiles = propagateDirtySignatures(existing.symbols, symbols, fileEntries);
        if (staleFiles.size > 0) {
            console.error(`[codesift] Dirty propagation: ${staleFiles.size} caller files marked stale`);
        }
    }
    // Build and cache BM25 index; invalidate code index cache
    const bm25 = buildBM25Index(symbols);
    bm25Indexes.set(repoName, bm25);
    codeIndexes.delete(repoName);
    // Sanity check: don't overwrite a complete index with a partial one
    // (WASM crash or walk failure can produce truncated results)
    const DROP_THRESHOLD = 0.5; // Reject if new index has <50% of old file count
    if (existing && fileEntries.length < existing.file_count * DROP_THRESHOLD && existing.file_count > 50) {
        console.error(`[codesift] SANITY CHECK FAILED for ${repoName}: ` +
            `new index has ${fileEntries.length} files vs ${existing.file_count} previously. ` +
            `Keeping old index. Use invalidate_cache + index_folder to force reindex.`);
        return {
            repo: repoName,
            root: rootPath,
            file_count: existing.file_count,
            symbol_count: existing.symbol_count,
            duration_ms: Date.now() - startTime,
        };
    }
    // Resolve workspaces (Task 7) — runs before persistence so collectImportEdges
    // and other downstream consumers see the populated `workspaces` field.
    // Gated behind CODESIFT_DISABLE_MONOREPO=1 kill switch (spec D-FB).
    let workspaces;
    if (process.env.CODESIFT_DISABLE_MONOREPO !== "1") {
        try {
            const { resolveWorkspaces } = await import("../storage/workspace-resolver.js");
            const resolved = await resolveWorkspaces(rootPath);
            if (resolved)
                workspaces = resolved.workspaces;
        }
        catch {
            // Resolver should never throw, but guard belt-and-braces — flat-repo
            // mode is the safe fallback.
        }
    }
    // Build and save code index
    const codeIndex = {
        repo: repoName,
        root: rootPath,
        symbols,
        files: fileEntries,
        created_at: Date.now(),
        updated_at: Date.now(),
        symbol_count: symbols.length,
        file_count: fileEntries.length,
        extractor_version: { ...EXTRACTOR_VERSIONS },
        ...(workspaces ? { workspaces } : {}),
    };
    await saveIndex(indexPath, codeIndex);
    // Embed symbols and chunks in background (non-fatal, don't block MCP response)
    // Large repos (71K symbols) can take minutes — fire-and-forget to prevent timeout
    embedSymbols(symbols, indexPath, repoName, config)
        .then(() => embedChunks(fileEntries, rootPath, repoName, indexPath, config, symbols))
        .catch((err) => {
        const msg = err instanceof Error ? err.message : String(err);
        console.error(`[codesift] Background embedding failed for ${repoName}: ${msg}`);
    });
    // Register in the global registry. If a stale entry exists with the same
    // root but a different name (e.g. `local/workspace` from before the git
    // origin auto-detect landed), drop it so `list_repos` doesn't show ghosts.
    const existingRepos = await listRegistryRepos(config.registryPath);
    for (const stale of existingRepos) {
        if (stale.root === rootPath && stale.name !== repoName) {
            await removeRepo(config.registryPath, stale.name);
            console.error(`[codesift] Migrated registry: ${stale.name} -> ${repoName} (same root)`);
        }
    }
    const meta = {
        name: repoName,
        root: rootPath,
        index_path: indexPath,
        symbol_count: symbols.length,
        file_count: fileEntries.length,
        updated_at: Date.now(),
    };
    await registerRepo(config.registryPath, meta);
    // Capture git HEAD for auto-refresh tracking
    try {
        const head = execFileSync("git", ["rev-parse", "HEAD"], {
            cwd: rootPath, encoding: "utf-8", timeout: 5000,
        }).trim();
        await updateRepoMeta(config.registryPath, repoName, { last_git_commit: head });
    }
    catch {
        // Not a git repo — skip
    }
    // Start file watcher for incremental updates (unless disabled)
    if (options?.watch !== false) {
        await setupWatcher(rootPath, repoName, indexPath);
    }
    // Auto-enable framework-specific tool bundles (NestJS, etc.)
    // Lazy import to avoid circular dep: index-tools → register-tools → tool handlers → index-tools
    try {
        const { detectFrameworks } = await import("../utils/framework-detect.js");
        const { enableFrameworkToolBundle } = await import("../register-tools.js");
        const tempIndex = { root: rootPath, files: fileEntries, symbols };
        const frameworks = detectFrameworks(tempIndex);
        for (const fw of frameworks) {
            const enabled = enableFrameworkToolBundle(fw);
            if (enabled.length > 0) {
                console.error(`[codesift] auto-enabled ${enabled.length} ${fw} tools for ${repoName}: ${enabled.join(", ")}`);
            }
        }
    }
    catch {
        // Non-fatal — framework auto-enable is a convenience feature
    }
    // Record completion timestamp so subsequent re-runs can short-circuit when
    // the watcher is keeping the index fresh.
    lastFullIndexAt.set(rootPath, Date.now());
    return {
        repo: repoName,
        root: rootPath,
        file_count: fileEntries.length,
        symbol_count: symbols.length,
        duration_ms: Date.now() - startTime,
    };
}
/**
 * Clone a remote git repository and index it.
 * Clones into ~/.codesift/repos/{name}. If already cloned, pulls latest.
 */
export async function indexRepo(url, options) {
    validateGitUrl(url);
    if (options?.branch) {
        validateGitRef(options.branch);
    }
    const config = loadConfig();
    const reposDir = join(config.dataDir, "repos");
    // Ensure repos directory exists (R-2: git clone requires parent to exist)
    await mkdirAsync(reposDir, { recursive: true });
    // Derive repo name from URL: "https://github.com/user/repo.git" → "repo"
    const urlBasename = basename(url).replace(/\.git$/, "");
    const cloneTarget = join(reposDir, urlBasename);
    // Check if already cloned — pull instead of clone
    let needsClone = true;
    try {
        const s = await stat(join(cloneTarget, ".git"));
        if (s.isDirectory()) {
            needsClone = false;
        }
    }
    catch {
        // Directory doesn't exist — will clone
    }
    // R-1: Use execFileSync (array form) to prevent shell injection.
    // execSync with string interpolation allows $(cmd) expansion in URLs.
    if (needsClone) {
        const args = ["clone", "--depth", "1"];
        if (options?.branch)
            args.push("--branch", options.branch);
        args.push("--", url, cloneTarget);
        execFileSync("git", args, { stdio: "pipe", timeout: GIT_CLONE_TIMEOUT_MS });
    }
    else {
        // Pull latest changes
        try {
            if (options?.branch) {
                execFileSync("git", ["-C", cloneTarget, "checkout", options.branch], {
                    stdio: "pipe",
                    timeout: GIT_CHECKOUT_TIMEOUT_MS,
                });
            }
            execFileSync("git", ["-C", cloneTarget, "pull", "--ff-only"], {
                stdio: "pipe",
                timeout: GIT_PULL_TIMEOUT_MS,
            });
        }
        catch (err) {
            // Pull may fail if detached HEAD — force fresh clone
            const message = err instanceof Error ? err.message : String(err);
            console.warn(`[codesift] Git pull failed for ${urlBasename}, re-cloning: ${message}`);
            await rm(cloneTarget, { recursive: true, force: true });
            const args = ["clone", "--depth", "1"];
            if (options?.branch)
                args.push("--branch", options.branch);
            args.push("--", url, cloneTarget);
            execFileSync("git", args, { stdio: "pipe", timeout: GIT_CLONE_TIMEOUT_MS });
        }
    }
    // Index the cloned repo (no watcher for remote repos)
    return indexFolder(cloneTarget, {
        include_paths: options?.include_paths,
        watch: false,
    });
}
/**
 * Maximum number of repos with active file watchers. Each chokidar watcher
 * holds at minimum a native FSEvents/inotify handle plus an fd per recursive
 * stream; in practice, bulk indexing dozens of repos with default watcher
 * behavior has exhausted the macOS system file table (ENFILE). Capping the
 * pool plus LRU eviction keeps fd usage bounded while still giving the
 * user-active repo (last touched) live incremental updates.
 *
 * Override via CODESIFT_MAX_WATCHERS env var. Set to 0 to disable watchers
 * entirely (suitable for CI / batch index runs).
 */
const DEFAULT_MAX_WATCHERS = 8;
function getMaxWatchers() {
    const env = process.env.CODESIFT_MAX_WATCHERS;
    if (env !== undefined) {
        const n = Number.parseInt(env, 10);
        if (Number.isFinite(n) && n >= 0)
            return n;
    }
    return DEFAULT_MAX_WATCHERS;
}
// Insertion-ordered Map gives us free LRU semantics: re-inserting a key
// moves it to the end, so the first key is the least-recently set up.
const watcherInsertionOrder = new Set();
/**
 * Replace or create a file watcher for incremental index updates.
 *
 * Enforces a max-active-watchers cap with LRU eviction so that bulk indexing
 * of many repos in one session cannot exhaust the system file table. When the
 * cap is exceeded the oldest watcher is closed; that repo's index becomes
 * stale-on-disk until the next explicit `index_folder` or `index_file`.
 */
async function setupWatcher(rootPath, repoName, indexPath) {
    const existingWatcher = activeWatchers.get(repoName);
    if (existingWatcher) {
        await stopWatcher(existingWatcher);
        watcherInsertionOrder.delete(repoName);
    }
    // Evict oldest watchers until we fit under the cap (with room for the new one)
    const maxWatchers = getMaxWatchers();
    if (maxWatchers === 0) {
        // Caller of indexFolder can also opt out via watch:false; this env is the
        // global kill-switch (e.g. for CI bulk-index jobs).
        return;
    }
    while (watcherInsertionOrder.size >= maxWatchers) {
        const oldest = watcherInsertionOrder.values().next().value;
        if (oldest === undefined)
            break;
        const oldWatcher = activeWatchers.get(oldest);
        if (oldWatcher) {
            await stopWatcher(oldWatcher).catch(() => { });
            activeWatchers.delete(oldest);
        }
        watcherInsertionOrder.delete(oldest);
        console.error(`[codesift] watcher cap reached (${maxWatchers}); evicted oldest watcher: ${oldest}`);
    }
    const watcher = await startWatcher(rootPath, (changedFile) => {
        handleFileChange(rootPath, repoName, indexPath, changedFile).catch((err) => {
            const message = err instanceof Error ? err.message : String(err);
            console.error(`[codesift] Watcher error for ${changedFile}: ${message}`);
        });
    }, (deletedFile) => {
        handleFileDelete(rootPath, repoName, indexPath, deletedFile).catch((err) => {
            const message = err instanceof Error ? err.message : String(err);
            console.error(`[codesift] Watcher delete error for ${deletedFile}: ${message}`);
        });
    });
    activeWatchers.set(repoName, watcher);
    watcherInsertionOrder.add(repoName);
}
/**
 * Handle a file change event from the watcher.
 * Re-parses the changed file and updates the index incrementally.
 */
async function handleFileChange(repoRoot, repoName, indexPath, relativeFile) {
    const fullPath = join(repoRoot, relativeFile);
    // Invalidate cached findings so the next scan sees the updated file contents.
    scanOnChanged(repoName, relativeFile);
    // Invalidate negative evidence for this file's subtree
    try {
        const { invalidateNegativeEvidence } = await import("../storage/session-state.js");
        invalidateNegativeEvidence(repoName, relativeFile);
    }
    catch {
        // Best-effort — session-state may not be loaded
    }
    // Invalidate Hono model cache for this file (canonicalized absolute path)
    try {
        const { honoCache } = await import("../cache/hono-cache.js");
        honoCache.invalidate(fullPath);
    }
    catch {
        // Best-effort — hono-cache may not be loaded
    }
    const result = await parseOneFile(fullPath, repoRoot, repoName);
    if (!result)
        return;
    await saveIncremental(indexPath, relativeFile, result.symbols, result.entry);
    if (loadConfig().secretScanEnabled) {
        try {
            await scanFileForSecrets(fullPath, relativeFile, repoName, result.symbols);
        }
        catch (err) {
            const message = err instanceof Error ? err.message : String(err);
            console.warn(`[codesift] Secret scan failed for ${relativeFile}: ${message}`);
        }
    }
    // Invalidate caches — lazy rebuild on next query via getBM25Index()
    bm25Indexes.delete(repoName);
    codeIndexes.delete(repoName);
    embeddingCaches.delete(repoName);
}
/**
 * Handle a file deletion event from the watcher.
 * Removes all symbols for the deleted file from the index.
 */
async function handleFileDelete(repoRoot, repoName, indexPath, relativeFile) {
    await removeFileFromIndex(indexPath, relativeFile);
    // Invalidate caches — lazy rebuild on next query via getBM25Index()
    bm25Indexes.delete(repoName);
    codeIndexes.delete(repoName);
    embeddingCaches.delete(repoName);
    scanOnDeleted(repoName, relativeFile);
    // Invalidate Hono model cache
    try {
        const { honoCache } = await import("../cache/hono-cache.js");
        honoCache.invalidate(join(repoRoot, relativeFile));
    }
    catch {
        // Best-effort
    }
}
export async function listAllRepos(options) {
    const config = loadConfig();
    let repos = await listRegistryRepos(config.registryPath);
    // Filter by name substring (case-insensitive)
    if (options?.name_contains) {
        const filter = options.name_contains.toLowerCase();
        repos = repos.filter((r) => r.name.toLowerCase().includes(filter));
    }
    if (options?.compact === false)
        return repos;
    // Default: ultra-compact — just repo names (agents only need the identifier)
    return repos.map((r) => r.name);
}
export async function invalidateCache(repoName) {
    const config = loadConfig();
    const meta = await getRepo(config.registryPath, repoName);
    if (!meta)
        return false;
    // Stop watcher
    const watcher = activeWatchers.get(repoName);
    if (watcher) {
        await stopWatcher(watcher);
        activeWatchers.delete(repoName);
    }
    // Remove in-memory caches
    bm25Indexes.delete(repoName);
    codeIndexes.delete(repoName);
    embeddingCaches.delete(repoName);
    // Delete index file + embedding files + chunk files
    const embeddingPath = getEmbeddingPath(meta.index_path);
    const embeddingMetaPath = getEmbeddingMetaPath(meta.index_path);
    const chunkPath = getChunkPath(meta.index_path);
    const chunkEmbeddingPath = getChunkEmbeddingPath(meta.index_path);
    const graphStorePath = getGraphPath(meta.index_path);
    for (const fp of [meta.index_path, embeddingPath, embeddingMetaPath, chunkPath, chunkEmbeddingPath, graphStorePath]) {
        try {
            await unlink(fp);
        }
        catch { /* File may not exist */ }
    }
    // Remove from registry
    await removeRepo(config.registryPath, repoName);
    return true;
}
/**
 * In-process record of the last indexed state per absolute file path.
 *
 * Telemetry (30d, 2026-06): 750 consecutive duplicate index_file calls at
 * avg 3.7s each (~47 min of agent wall-clock). Two causes: (1) duplicate
 * hook registrations firing index_file twice per edit, and (2) a race where
 * call N+1's on-disk mtime pre-check read the index before call N's
 * serialized saveIncremental landed, forcing a full re-parse + full-index
 * save. This map short-circuits both in-process in ~1ms (mtime first, then
 * content hash for touch/no-op rewrites) without loading the on-disk index.
 */
const lastIndexedState = new Map();
/** Test hook — clear the in-process last-indexed state. */
export function clearLastIndexedStateForTesting() {
    lastIndexedState.clear();
}
/**
 * Re-index a single file instantly. Finds the repo by matching the file
 * path against indexed repo roots. Updates symbols, BM25 index, and
 * invalidates embedding cache — no full repo walk needed.
 */
export async function indexFile(filePath) {
    const absPath = resolve(filePath);
    const config = loadConfig();
    const repos = await listRegistryRepos(config.registryPath);
    // Find the most specific repo root that contains this file
    const matchingRepo = repos
        .filter((r) => absPath.startsWith(r.root + "/") || absPath === r.root)
        .sort((a, b) => b.root.length - a.root.length)[0];
    if (!matchingRepo) {
        throw new Error(`No indexed repo contains "${absPath}". Run index_folder first.`);
    }
    const startTime = Date.now();
    const relPath = relative(matchingRepo.root, absPath);
    // If the changed file is a TS/JS config that drives path resolution, drop
    // caches so incremental indexing picks up new `paths` / `extends`.
    {
        const cfg = basename(absPath).toLowerCase();
        if ((cfg.startsWith("tsconfig") || cfg.startsWith("jsconfig")) &&
            cfg.endsWith(".json")) {
            clearTsconfigCache();
        }
    }
    // In-process short-circuit: mtime, then content hash. Both avoid loading
    // the on-disk index entirely (the expensive part on large repos).
    const st = await stat(absPath);
    const mem = lastIndexedState.get(absPath);
    if (mem && Math.round(st.mtimeMs) === mem.mtimeMs) {
        return {
            repo: matchingRepo.name,
            file: relPath,
            symbol_count: mem.symbolCount,
            duration_ms: Date.now() - startTime,
            skipped: true,
        };
    }
    const content = await readFile(absPath, "utf-8").catch(() => null);
    const contentHash = content !== null ? createHash("sha1").update(content).digest("hex") : null;
    if (mem && contentHash !== null && contentHash === mem.contentHash) {
        // Touched / rewritten with identical content — refresh mtime, skip work.
        mem.mtimeMs = Math.round(st.mtimeMs);
        return {
            repo: matchingRepo.name,
            file: relPath,
            symbol_count: mem.symbolCount,
            duration_ms: Date.now() - startTime,
            skipped: true,
        };
    }
    // On-disk mtime check — first touch of this file in this process (CLI
    // hook invocations, fresh server). Skips files unchanged since the last
    // full index, and seeds the in-process state for subsequent calls.
    if (!mem) {
        const existing = await loadIndex(matchingRepo.index_path);
        if (existing) {
            const prevEntry = existing.files.find((f) => f.path === relPath);
            if (prevEntry?.mtime_ms && Math.round(st.mtimeMs) === prevEntry.mtime_ms) {
                if (contentHash !== null) {
                    lastIndexedState.set(absPath, {
                        mtimeMs: Math.round(st.mtimeMs),
                        contentHash,
                        symbolCount: prevEntry.symbol_count,
                    });
                }
                return {
                    repo: matchingRepo.name,
                    file: relPath,
                    symbol_count: prevEntry.symbol_count,
                    duration_ms: Date.now() - startTime,
                    skipped: true,
                };
            }
        }
    }
    const result = await parseOneFile(absPath, matchingRepo.root, matchingRepo.name);
    if (!result) {
        throw new Error(`Failed to parse "${relPath}"`);
    }
    await saveIncremental(matchingRepo.index_path, relPath, result.symbols, result.entry);
    if (contentHash !== null) {
        lastIndexedState.set(absPath, {
            mtimeMs: Math.round(st.mtimeMs),
            contentHash,
            symbolCount: result.symbols.length,
        });
    }
    let secretFindingsCount = 0;
    if (config.secretScanEnabled) {
        try {
            secretFindingsCount = (await scanFileForSecrets(absPath, relPath, matchingRepo.name, result.symbols)).length;
        }
        catch (err) {
            const message = err instanceof Error ? err.message : String(err);
            console.warn(`[codesift] Secret scan failed for ${relPath}: ${message}`);
        }
    }
    // Invalidate caches — lazy rebuild on next query via getBM25Index()
    bm25Indexes.delete(matchingRepo.name);
    codeIndexes.delete(matchingRepo.name);
    embeddingCaches.delete(matchingRepo.name);
    let secretsWarning;
    if (secretFindingsCount > 0) {
        secretsWarning = `\u26A0 ${secretFindingsCount} potential secret(s) detected`;
    }
    return {
        repo: matchingRepo.name,
        file: relPath,
        symbol_count: result.symbols.length,
        duration_ms: Date.now() - startTime,
        ...(secretsWarning ? { secrets_warning: secretsWarning } : {}),
    };
}
// ---------------------------------------------------------------------------
// Git-based auto-refresh — transparent freshness check before index access
// ---------------------------------------------------------------------------
const freshnessChecked = new Map();
const FRESHNESS_INTERVAL_MS = 60_000;
const MAX_DIFF_FILES = 50;
/**
 * Ensure the index for a repo is fresh relative to git HEAD.
 * Throttled to once per minute per repo. Reindexes changed files if HEAD moved.
 * No-op for non-git repos.
 */
export async function ensureIndexFresh(repoName) {
    const lastCheck = freshnessChecked.get(repoName);
    if (lastCheck && Date.now() - lastCheck < FRESHNESS_INTERVAL_MS) {
        return { status: "fresh" };
    }
    const config = loadConfig();
    const meta = await getRepo(config.registryPath, repoName);
    if (!meta)
        return { status: "skipped" };
    let currentCommit;
    try {
        currentCommit = execFileSync("git", ["rev-parse", "HEAD"], {
            cwd: meta.root, encoding: "utf-8", timeout: 5000,
        }).trim();
    }
    catch {
        freshnessChecked.set(repoName, Date.now());
        return { status: "skipped" };
    }
    if (meta.last_git_commit === currentCommit) {
        freshnessChecked.set(repoName, Date.now());
        return { status: "fresh" };
    }
    // HEAD moved — find changed files
    let changedFiles = [];
    if (meta.last_git_commit) {
        try {
            const diff = execFileSync("git", [
                "diff", "--name-only", "--diff-filter=ACMR",
                `${meta.last_git_commit}..${currentCommit}`,
            ], {
                cwd: meta.root, encoding: "utf-8", timeout: 10_000,
            });
            changedFiles = diff.trim().split("\n").filter(Boolean);
        }
        catch {
            // Stored commit gone (rebase/squash) — will do full incremental
            changedFiles = [];
        }
    }
    if (changedFiles.length > 0 && changedFiles.length <= MAX_DIFF_FILES) {
        for (const file of changedFiles) {
            try {
                await indexFile(join(meta.root, file));
            }
            catch {
                // File deleted or unparseable — skip
            }
        }
    }
    else if (changedFiles.length > MAX_DIFF_FILES || !meta.last_git_commit) {
        await indexFolder(meta.root, { incremental: true, watch: false });
    }
    await updateRepoMeta(config.registryPath, repoName, {
        last_git_commit: currentCommit,
        updated_at: Date.now(),
    });
    bm25Indexes.delete(repoName);
    codeIndexes.delete(repoName);
    embeddingCaches.delete(repoName);
    freshnessChecked.set(repoName, Date.now());
    return { status: "refreshed", files_updated: changedFiles.length };
}
/** Reset freshness throttle cache. Exported for testing. */
export function resetFreshnessCache() {
    freshnessChecked.clear();
}
/** Stop every active watcher and clear the in-memory index caches. Exported
 *  for testing — afterAll teardown of integration tests that index temp dirs
 *  must call this before `rm -rf`-ing the temp dir, otherwise the chokidar
 *  watcher races the rm and emits ENOTEMPTY/ENOENT noise that shows up as
 *  file-level test failures even when every test inside passed. */
export async function stopAllWatchersForTesting() {
    const watchers = [...activeWatchers.values()];
    activeWatchers.clear();
    await Promise.all(watchers.map((w) => stopWatcher(w).catch(() => { })));
    bm25Indexes.clear();
    codeIndexes.clear();
    embeddingCaches.clear();
    freshnessChecked.clear();
}
// ---------------------------------------------------------------------------
// Index access — with auto-refresh
// ---------------------------------------------------------------------------
/**
 * Get the in-memory BM25 index for a repo.
 * Loads from disk if not cached. Auto-refreshes if git HEAD moved.
 */
export async function getBM25Index(repoName) {
    await ensureIndexFresh(repoName);
    const cached = bm25Indexes.get(repoName);
    if (cached)
        return cached;
    const config = loadConfig();
    const meta = await getRepo(config.registryPath, repoName);
    if (!meta)
        return null;
    const index = await loadIndex(meta.index_path);
    if (!index)
        return null;
    const bm25 = buildBM25Index(index.symbols);
    bm25Indexes.set(repoName, bm25);
    return bm25;
}
/**
 * Get the code index for a repo from disk.
 * Starts watcher if not running (lazy start after server restart).
 */
/**
 * Get the code index for a repo from disk. Auto-refreshes if git HEAD moved.
 */
export async function getCodeIndex(repoName, options) {
    const config = loadConfig();
    const resolved = await resolveRegisteredRepoMeta(config.registryPath, repoName);
    if (!resolved)
        return null;
    const { resolvedName, meta } = resolved;
    if (!options?.skipFreshness) {
        await ensureIndexFresh(resolvedName);
    }
    const cached = codeIndexes.get(resolvedName);
    if (cached)
        return cached;
    const result = await loadIndexOrStale(meta.index_path, { ...EXTRACTOR_VERSIONS });
    if (!result)
        return null;
    if (result.status === "stale") {
        const extra = result.mismatch_detail ? ` — ${result.mismatch_detail}` : "";
        console.warn(`[codesift] stale index for ${resolvedName}: extractor_version_mismatch ` +
            `(${result.language} expected ${result.expected_version}, got ${result.actual_version})${extra}. ` +
            `Run index_folder to refresh.`);
        return null;
    }
    codeIndexes.set(resolvedName, result.index);
    return result.index;
}
/**
 * Walk up from dir until a .git directory is found. Returns the git root or null.
 */
async function findGitRoot(dir) {
    let current = resolve(dir);
    while (true) {
        try {
            await stat(join(current, ".git"));
            return current;
        }
        catch {
            const parent = dirname(current);
            if (parent === current)
                return null;
            current = parent;
        }
    }
}
/**
 * Called at server startup. If the CWD is inside a git repo that isn't indexed yet,
 * index it automatically in the background so tools work without manual setup.
 */
export async function autoIndexCurrentRepo(cwd) {
    const gitRoot = await findGitRoot(cwd);
    if (!gitRoot)
        return;
    const repoName = getRepoName(gitRoot);
    const config = loadConfig();
    const existing = await getRepo(config.registryPath, repoName);
    if (existing)
        return;
    console.error(`[codesift] Auto-indexing ${repoName} (first use)...`);
    await indexFolder(gitRoot);
    console.error(`[codesift] Auto-index complete: ${repoName}`);
}
/**
 * Get the in-memory embedding cache for a repo.
 * Loads from disk if not cached. Returns null if no embeddings file exists.
 */
export async function getEmbeddingCache(repoName) {
    const cached = embeddingCaches.get(repoName);
    if (cached)
        return cached;
    const config = loadConfig();
    const meta = await getRepo(config.registryPath, repoName);
    if (!meta)
        return null;
    const embeddingPath = getEmbeddingPath(meta.index_path);
    const embeddings = await loadEmbeddings(embeddingPath);
    if (embeddings.size === 0)
        return null;
    embeddingCaches.set(repoName, embeddings);
    return embeddings;
}
// ---------------------------------------------------------------------------
// Astro extractor version check + lockfile re-index
// ---------------------------------------------------------------------------
export const ASTRO_LOCK_FILENAME = "astro-reindex.lock";
export const EXTRACTOR_VERSIONS_FILENAME = "extractor-versions.json";
const LOCK_STALE_MS = 60_000; // 60 seconds
/**
 * Check if the stored astro extractor version matches the current one.
 * If not, re-extract all .astro files with lockfile protection.
 *
 * @param dataDir   The data directory (e.g., ~/.codesift or a test tmpdir)
 * @param repoRoot  The repo root path (for locating .astro files)
 * @param astroFiles  Relative paths to .astro files in the repo
 */
export async function checkAstroExtractorVersion(dataDir, repoRoot, astroFiles) {
    // Read stored version snapshot
    const versionsPath = join(dataDir, EXTRACTOR_VERSIONS_FILENAME);
    let storedVersions = {};
    try {
        const raw = await readFile(versionsPath, "utf-8");
        storedVersions = JSON.parse(raw);
    }
    catch {
        // File doesn't exist or is invalid — treat as version mismatch
    }
    // Check if astro version matches
    if (storedVersions.astro === EXTRACTOR_VERSIONS.astro) {
        return { reindexed: false, reason: "astro extractor up to date" };
    }
    if (astroFiles.length === 0) {
        // No astro files to re-extract — just update the version snapshot
        await writeVersionSnapshot(dataDir, storedVersions);
        return { reindexed: false, reason: "no astro files to re-extract" };
    }
    // Acquire lockfile
    const lockPath = join(dataDir, ASTRO_LOCK_FILENAME);
    if (!acquireLock(lockPath)) {
        return { reindexed: false, reason: "astro re-index in progress (locked by another process)" };
    }
    try {
        // Re-extract all .astro files
        let count = 0;
        for (const relPath of astroFiles) {
            const absPath = join(repoRoot, relPath);
            try {
                const result = await parseOneFile(absPath, repoRoot, "");
                if (result) {
                    count++;
                    if (count % 100 === 0) {
                        console.error(`[codesift] Astro re-index progress: ${count}/${astroFiles.length} files`);
                    }
                }
            }
            catch {
                // File may have been deleted or is unparseable
            }
        }
        // Write version snapshot atomically
        await writeVersionSnapshot(dataDir, storedVersions);
        return {
            reindexed: true,
            files_reindexed: count,
            reason: `re-extracted ${count} astro files (version ${storedVersions.astro ?? "none"} → ${EXTRACTOR_VERSIONS.astro})`,
        };
    }
    finally {
        // Release lockfile
        try {
            unlinkSync(lockPath);
        }
        catch {
            // Lock may have been cleaned up already
        }
    }
}
/**
 * Acquire an exclusive lockfile. Returns true if lock was acquired.
 * If lockfile exists but is stale (mtime > 60s), deletes it and retries once.
 */
function acquireLock(lockPath) {
    try {
        const fd = openSync(lockPath, "wx");
        closeSync(fd);
        return true;
    }
    catch (err) {
        if (err.code !== "EEXIST") {
            return false;
        }
        // Lock exists — check if stale
        try {
            const lockStat = statSync(lockPath);
            const age = Date.now() - lockStat.mtimeMs;
            if (age > LOCK_STALE_MS) {
                // Stale lock — delete and retry once
                unlinkSync(lockPath);
                try {
                    const fd = openSync(lockPath, "wx");
                    closeSync(fd);
                    return true;
                }
                catch {
                    return false;
                }
            }
        }
        catch {
            // stat failed — lock was just removed by another process
            return false;
        }
        return false;
    }
}
/**
 * Atomically write the extractor versions snapshot.
 * Uses write-to-tmp + rename for crash safety.
 */
async function writeVersionSnapshot(dataDir, existingVersions) {
    const versionsPath = join(dataDir, EXTRACTOR_VERSIONS_FILENAME);
    const tmpPath = `${versionsPath}.tmp.${Date.now()}`;
    const snapshot = { ...existingVersions, astro: EXTRACTOR_VERSIONS.astro };
    writeFileSync(tmpPath, JSON.stringify(snapshot), "utf-8");
    renameSync(tmpPath, versionsPath);
}
//# sourceMappingURL=index-tools.js.map