/**
 * Sparse N-gram Index Engine
 *
 * Fast text search using trigram decomposition with probabilistic masks,
 * inspired by Cursor's fast regex search and GitHub's Project Blackbird.
 *
 * Instead of scanning every document, queries decompose into trigrams,
 * look up posting lists, intersect them, and only full-match on candidates.
 * Probabilistic masks (locMask + nextMask) provide "3.5-gram" specificity
 * from 3-gram keys, dramatically reducing false positives.
 *
 * Persistence: SQLite (same pattern as memory.db / graph.db)
 */
import Database from 'better-sqlite3';
import * as path from 'path';
import * as fs from 'fs';
import { resolveNexusStateDir } from './runtime-registry.js';
const _ngramNoticeOnce = new Set();
function logNgramNoticeOnce(key, message) {
    if (_ngramNoticeOnce.has(key))
        return;
    _ngramNoticeOnce.add(key);
    console.error(message);
}
function readEnvBytes(name, fallback) {
    const raw = process.env[name];
    if (!raw)
        return fallback;
    const parsed = Number(raw);
    if (!Number.isFinite(parsed) || parsed <= 0)
        return fallback;
    return Math.floor(parsed);
}
function safeStatSize(filePath) {
    try {
        return fs.existsSync(filePath) ? fs.statSync(filePath).size : 0;
    }
    catch {
        return 0;
    }
}
// ─────────────────────────────────────────────────────────────────────────────
// SQLite WAL footprint helpers — used by quota guard, rotation, and cleanup.
// SQLite databases are db + db-wal + db-shm. Any size accounting that ignores
// wal/shm misses the bug that turned ngram-index.db-wal into 84GB on disk.
// ─────────────────────────────────────────────────────────────────────────────
const NGRAM_DEFAULT_WAL_LIMIT_BYTES = 64 * 1024 * 1024; // 64 MB
export const NGRAM_WARMUP_MAX_DB_BYTES = 200 * 1024 * 1024; // 200 MB
export const NGRAM_DEFAULT_ROTATE_BYTES = NGRAM_WARMUP_MAX_DB_BYTES;
export const NGRAM_DEFAULT_FOOTPRINT_BYTES = NGRAM_WARMUP_MAX_DB_BYTES;
const NGRAM_DEFAULT_CHECKPOINT_INTERVAL_MS = 30_000;
const NGRAM_DEFAULT_CHECKPOINT_DOC_COUNT = 200;
export function getNgramWalPath(dbPath) {
    return `${dbPath}-wal`;
}
export function getNgramShmPath(dbPath) {
    return `${dbPath}-shm`;
}
export function getNgramFootprintBytes(dbPath) {
    return safeStatSize(dbPath)
        + safeStatSize(getNgramWalPath(dbPath))
        + safeStatSize(getNgramShmPath(dbPath));
}
// ─────────────────────────────────────────────────────────────────────────────
// Character pair frequency weights (precomputed from common code patterns)
// Rarer pairs get higher weights for sparse n-gram extraction.
// ─────────────────────────────────────────────────────────────────────────────
const CHAR_PAIR_WEIGHTS = new Map();
function crc32Single(value) {
    let crc = value;
    for (let i = 0; i < 8; i++) {
        crc = (crc >>> 1) ^ (crc & 1 ? 0xEDB88320 : 0);
    }
    return crc >>> 0;
}
function charPairHash(a, b) {
    const val = (a.charCodeAt(0) << 8) | b.charCodeAt(0);
    return crc32Single(val) >>> 0;
}
/**
 * Weight for a character pair — higher = rarer = more selective.
 * Uses CRC32 hash to deterministically assign weights.
 * Common pairs (e.g., "th", "he", "in") get lower weights.
 */
function charPairWeight(a, b) {
    const hash = charPairHash(a, b);
    const cached = CHAR_PAIR_WEIGHTS.get(hash);
    if (cached !== undefined)
        return cached;
    // Deterministic weight from hash, range [0.0, 1.0]
    const weight = (hash % 1000) / 1000;
    CHAR_PAIR_WEIGHTS.set(hash, weight);
    return weight;
}
// ─────────────────────────────────────────────────────────────────────────────
// Trigram extraction
// ─────────────────────────────────────────────────────────────────────────────
function trigramHash(trigram) {
    // FNV-1a inspired hash for 3-char trigrams
    let hash = 0x811c9dc5;
    for (let i = 0; i < trigram.length; i++) {
        hash ^= trigram.charCodeAt(i);
        hash = Math.imul(hash, 0x01000193);
    }
    return hash >>> 0;
}
function extractTrigrams(text) {
    const lower = text.toLowerCase();
    const entries = [];
    for (let i = 0; i <= lower.length - 3; i++) {
        const trigram = lower.substring(i, i + 3);
        // Skip trigrams that are all whitespace
        if (/^\s+$/.test(trigram))
            continue;
        entries.push({
            trigram,
            hash: trigramHash(trigram),
            position: i,
            nextChar: i + 3 < lower.length ? lower[i + 3] : '',
        });
    }
    return entries;
}
/**
 * Extract sparse n-grams: variable-length n-grams where edge character-pair
 * weights exceed all internal weights. This produces fewer, more selective
 * index keys for rarer character sequences.
 */
function extractSparseNgrams(text) {
    const lower = text.toLowerCase();
    if (lower.length < 3)
        return [];
    const results = [];
    let i = 0;
    while (i < lower.length - 2) {
        const startWeight = charPairWeight(lower[i], lower[i + 1]);
        let end = i + 2;
        // Extend while internal pair weights are lower than start weight
        while (end < lower.length) {
            const internalWeight = charPairWeight(lower[end - 1], lower[end]);
            if (internalWeight >= startWeight)
                break;
            end++;
        }
        const ngram = lower.substring(i, end + 1 > lower.length ? lower.length : end);
        if (ngram.length >= 3 && !/^\s+$/.test(ngram)) {
            results.push({
                ngram,
                hash: trigramHash(ngram.substring(0, 3)), // index by first trigram
                position: i,
            });
        }
        i = end > i + 1 ? end : i + 1;
    }
    return results;
}
// ─────────────────────────────────────────────────────────────────────────────
// Probabilistic masks
// ─────────────────────────────────────────────────────────────────────────────
/** Build position mask: which of 8 slots this trigram appears at */
function buildLocMask(position) {
    return 1 << (position % 8);
}
/** Build next-char bloom filter: hash the following character into 8 bits */
function buildNextMask(nextChar) {
    if (!nextChar)
        return 0xFF; // wildcard — matches anything
    return 1 << (nextChar.charCodeAt(0) % 8);
}
export class NgramIndex {
    db;
    dbPath;
    storageDirty = false;
    deferWarm;
    // In-memory lookup: trigramHash → true (for fast existence check)
    knownHashes = new Set();
    // Prepared statements
    insertStmt;
    deleteStmt;
    lookupStmt;
    docExistsStmt;
    // Write-side checkpoint accounting (Phase 2): bound WAL growth on hot indexers.
    writesSinceCheckpoint = 0;
    lastCheckpointAt = 0;
    quotaSkipNoticeShown = false;
    // Search analytics
    _searchStats = {
        totalQueries: 0,
        totalTokensSaved: 0,
        totalResultsReturned: 0,
        recentQueries: [],
    };
    constructor(dbPath, options) {
        const stateDir = resolveNexusStateDir();
        this.dbPath = dbPath ?? path.join(stateDir, 'ngram-index.db');
        this.deferWarm = options?.deferWarm ?? false;
        this.rotateOversizeDbIfNeeded();
        this.db = new Database(this.dbPath);
        this.initSchema();
        this.prepareStatements();
        this.warmHashSet();
        try {
            const sizeBytes = safeStatSize(this.dbPath);
            // Only VACUUM on medium-size DBs. Large DBs take too long
            // synchronously — VACUUM rewrites the entire file. Skip here; let
            // periodic maintenance handle it.
            if (sizeBytes > 32 * 1024 * 1024 && sizeBytes <= NGRAM_WARMUP_MAX_DB_BYTES) {
                this.optimizeStorage(true);
            }
            else if (sizeBytes > NGRAM_WARMUP_MAX_DB_BYTES) {
                logNgramNoticeOnce(`ngram:vacuum-skip:${this.dbPath}`, `[NgramIndex] skipping VACUUM on large DB (${Math.round(sizeBytes / 1024 / 1024)}MB) db=${this.dbPath}`);
            }
        }
        catch {
            // Best effort startup optimization for oversized indexes.
        }
    }
    rotateOversizeDbIfNeeded() {
        // Count the full SQLite footprint (db + wal + shm). The 84GB regression
        // happened because a 32MB db file had a 84GB -wal sibling that this
        // routine never inspected.
        const rotateBytes = readEnvBytes('NEXUS_NGRAM_ROTATE_BYTES', NGRAM_DEFAULT_ROTATE_BYTES);
        const dbBytes = safeStatSize(this.dbPath);
        const footprint = getNgramFootprintBytes(this.dbPath);
        if (footprint <= 0 || footprint < rotateBytes)
            return;
        // The n-gram DB is a rebuildable cache, so oversized rotation drops it by
        // default. Operators can opt into one retained archive for forensics with
        // NEXUS_NGRAM_ARCHIVE_OVERSIZE=1.
        const dir = path.dirname(this.dbPath);
        const base = path.basename(this.dbPath);
        const archiveEnabled = process.env.NEXUS_NGRAM_ARCHIVE_OVERSIZE === '1';
        const existing = fs.existsSync(dir)
            ? fs.readdirSync(dir).filter((entry) => entry.startsWith(`${base}.oversize.`)).sort().reverse()
            : [];
        const keepCount = archiveEnabled ? 1 : 0;
        for (const entry of existing.slice(keepCount)) {
            try {
                fs.rmSync(path.join(dir, entry), { force: true });
            }
            catch { /* best effort */ }
        }
        const stamp = Date.now();
        const rotatedPath = `${this.dbPath}.oversize.${stamp}`;
        const removeSibling = (suffix) => {
            try {
                fs.rmSync(`${this.dbPath}${suffix}`, { force: true });
            }
            catch { /* best effort */ }
        };
        try {
            if (archiveEnabled) {
                fs.renameSync(this.dbPath, rotatedPath);
            }
            else {
                fs.rmSync(this.dbPath, { force: true });
            }
            // db is gone; the WAL/SHM siblings are stale and unsafe to keep — drop
            // them so SQLite never re-attaches them on the next open.
            removeSibling('-wal');
            removeSibling('-shm');
            logNgramNoticeOnce(`ngram:rotated:${this.dbPath}:${stamp}`, `[NgramIndex] rotated oversized DB (footprint=${Math.round(footprint / 1024 / 1024)}MB db=${Math.round(dbBytes / 1024 / 1024)}MB threshold=${Math.round(rotateBytes / 1024 / 1024)}MB archive=${archiveEnabled ? 'on' : 'off'}) db=${this.dbPath}${archiveEnabled ? ` rotated=${rotatedPath}` : ''}`);
        }
        catch (err) {
            logNgramNoticeOnce(`ngram:rotate-failed:${this.dbPath}`, `[NgramIndex] oversize rotation failed; continuing with existing DB db=${this.dbPath} err=${String(err?.message ?? err)}`);
        }
    }
    initSchema() {
        const walLimitBytes = readEnvBytes('NEXUS_NGRAM_WAL_LIMIT_BYTES', NGRAM_DEFAULT_WAL_LIMIT_BYTES);
        this.db.pragma('journal_mode = WAL');
        this.db.pragma('synchronous = NORMAL');
        this.db.pragma('cache_size = -16000');
        this.db.pragma('busy_timeout = 5000');
        // Bound WAL growth: autocheckpoint every N pages and hard-cap journal size
        // so it never balloons (the 84GB regression). temp_store=MEMORY avoids
        // spilling sort/temp tables into another on-disk file. mmap_size=0
        // disables the OS-mapped read cache that can keep pages alive on small VMs.
        this.db.pragma('wal_autocheckpoint = 1000');
        this.db.pragma(`journal_size_limit = ${walLimitBytes}`);
        this.db.pragma('temp_store = MEMORY');
        this.db.pragma('mmap_size = 0');
        this.db.exec(`
      CREATE TABLE IF NOT EXISTS ngram_postings (
        ngram_hash INTEGER NOT NULL,
        doc_id TEXT NOT NULL,
        loc_mask INTEGER NOT NULL DEFAULT 0,
        next_mask INTEGER NOT NULL DEFAULT 0,
        PRIMARY KEY (ngram_hash, doc_id)
      );

      CREATE INDEX IF NOT EXISTS idx_postings_doc ON ngram_postings(doc_id);
      CREATE INDEX IF NOT EXISTS idx_postings_hash ON ngram_postings(ngram_hash);

      CREATE TABLE IF NOT EXISTS ngram_docs (
        doc_id TEXT PRIMARY KEY,
        text_length INTEGER NOT NULL DEFAULT 0,
        indexed_at INTEGER NOT NULL DEFAULT 0
      );
    `);
    }
    prepareStatements() {
        this.insertStmt = this.db.prepare(`
      INSERT OR REPLACE INTO ngram_postings (ngram_hash, doc_id, loc_mask, next_mask)
      VALUES (?, ?, ?, ?)
    `);
        this.deleteStmt = this.db.prepare(`
      DELETE FROM ngram_postings WHERE doc_id = ?
    `);
        this.lookupStmt = this.db.prepare(`
      SELECT doc_id AS docId, loc_mask AS locMask, next_mask AS nextMask
      FROM ngram_postings WHERE ngram_hash = ?
    `);
        this.docExistsStmt = this.db.prepare(`
      SELECT 1 FROM ngram_docs WHERE doc_id = ?
    `);
    }
    /** Load known trigram hashes into memory for fast existence checks.
     *  Call warmHashSetDeferred() later if init was skipped. */
    warmHashSet() {
        // Check explicit defer flag (set by caller or env var).
        const envDefer = process.env.NEXUS_NGRAM_DEFER_WARM === '1';
        if (this.deferWarm || envDefer) {
            logNgramNoticeOnce(`ngram:warmup-defer:${this.dbPath}`, `[NgramIndex] warmup deferred (deferWarm=${this.deferWarm} env=${envDefer}) db=${this.dbPath}`);
            return;
        }
        // Size guard: SELECT DISTINCT on very large DBs blocks startup under swap pressure.
        try {
            const sizeBytes = fs.statSync(this.dbPath).size;
            if (sizeBytes > NGRAM_WARMUP_MAX_DB_BYTES) {
                logNgramNoticeOnce(`ngram:warmup-skip:${this.dbPath}`, `[NgramIndex] warmup skipped — DB too large (${Math.round(sizeBytes / 1024 / 1024)}MB > ${Math.round(NGRAM_WARMUP_MAX_DB_BYTES / 1024 / 1024)}MB) db=${this.dbPath}`);
                return;
            }
        }
        catch { /* stat failed — proceed with warmup */ }
        const rows = this.db.prepare('SELECT DISTINCT ngram_hash FROM ngram_postings LIMIT 50000').all();
        for (const row of rows) {
            this.knownHashes.add(row.ngram_hash);
        }
    }
    /** Warm the hash set after a deferred init. No-op if already warm. */
    warmHashSetDeferred() {
        if (this.knownHashes.size > 0)
            return;
        const rows = this.db.prepare('SELECT DISTINCT ngram_hash FROM ngram_postings LIMIT 50000').all();
        for (const row of rows) {
            this.knownHashes.add(row.ngram_hash);
        }
    }
    // ── Document Management ─────────────────────────────────────────────────
    /** Run an opportunistic WAL checkpoint. PASSIVE first, escalate to TRUNCATE
     *  if the WAL is still over its configured limit. Returns the WAL byte size
     *  observed after the attempt so callers can report progress. */
    checkpointIfNeeded(reason) {
        const walLimit = readEnvBytes('NEXUS_NGRAM_WAL_LIMIT_BYTES', NGRAM_DEFAULT_WAL_LIMIT_BYTES);
        try {
            this.db.exec('PRAGMA wal_checkpoint(PASSIVE)');
        }
        catch {
            // best effort
        }
        let walBytes = this.getWalBytes();
        if (walBytes > walLimit || reason !== 'periodic') {
            try {
                this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
            }
            catch {
                // best effort
            }
            walBytes = this.getWalBytes();
        }
        this.writesSinceCheckpoint = 0;
        this.lastCheckpointAt = Date.now();
        return walBytes;
    }
    /** Footprint quota check before a write. Returns true if the write should
     *  proceed; false if it should be skipped. Throws when NEXUS_NGRAM_STRICT_QUOTA=1
     *  so callers can surface hard quota failures during tests/CI. */
    allowWrite() {
        const walLimit = readEnvBytes('NEXUS_NGRAM_WAL_LIMIT_BYTES', NGRAM_DEFAULT_WAL_LIMIT_BYTES);
        const maxFootprint = readEnvBytes('NEXUS_NGRAM_MAX_FOOTPRINT_BYTES', NGRAM_DEFAULT_FOOTPRINT_BYTES);
        const strict = process.env.NEXUS_NGRAM_STRICT_QUOTA === '1';
        let walBytes = this.getWalBytes();
        if (walBytes > walLimit) {
            walBytes = this.checkpointIfNeeded('quota');
        }
        const footprint = this.getSqliteFootprintBytes();
        if (footprint > maxFootprint) {
            const msg = `[NgramIndex] footprint quota exceeded — db=${this.dbPath} footprint=${footprint} cap=${maxFootprint} wal=${walBytes}`;
            if (strict) {
                throw new Error(msg);
            }
            if (!this.quotaSkipNoticeShown) {
                this.quotaSkipNoticeShown = true;
                logNgramNoticeOnce(`ngram:quota-skip:${this.dbPath}`, msg);
            }
            return false;
        }
        // Recovered — re-arm the warning so a later breach prints again.
        this.quotaSkipNoticeShown = false;
        return true;
    }
    /** Index a document's text content */
    addDocument(docId, text) {
        if (!this.allowWrite())
            return;
        // Remove existing postings for this doc (idempotent)
        this.deleteStmt.run(docId);
        const trigrams = extractTrigrams(text);
        // Aggregate postings per trigram hash: OR together masks
        const aggregated = new Map();
        for (const entry of trigrams) {
            const existing = aggregated.get(entry.hash);
            const locMask = buildLocMask(entry.position);
            const nextMask = buildNextMask(entry.nextChar);
            if (existing) {
                existing.locMask |= locMask;
                existing.nextMask |= nextMask;
            }
            else {
                aggregated.set(entry.hash, { locMask, nextMask });
            }
        }
        // Batch insert
        const insertMany = this.db.transaction(() => {
            for (const [hash, masks] of aggregated) {
                this.insertStmt.run(hash, docId, masks.locMask, masks.nextMask);
                this.knownHashes.add(hash);
            }
            this.db.prepare(`
        INSERT OR REPLACE INTO ngram_docs (doc_id, text_length, indexed_at)
        VALUES (?, ?, ?)
      `).run(docId, text.length, Date.now());
        });
        insertMany();
        this.maybePeriodicCheckpoint();
    }
    /** Trigger a passive checkpoint when either the doc or time interval threshold
     *  is hit. Cheap on a healthy DB, prevents WAL growth on bulk indexing. */
    maybePeriodicCheckpoint() {
        this.writesSinceCheckpoint += 1;
        const intervalMs = readEnvBytes('NEXUS_NGRAM_CHECKPOINT_INTERVAL_MS', NGRAM_DEFAULT_CHECKPOINT_INTERVAL_MS);
        const docThreshold = readEnvBytes('NEXUS_NGRAM_CHECKPOINT_DOCS', NGRAM_DEFAULT_CHECKPOINT_DOC_COUNT);
        const dueByDocs = this.writesSinceCheckpoint >= docThreshold;
        const dueByTime = Date.now() - this.lastCheckpointAt > intervalMs;
        if (!dueByDocs && !dueByTime)
            return;
        this.checkpointIfNeeded('periodic');
    }
    /** Remove a document from the index */
    removeDocument(docId) {
        this.db.transaction(() => {
            this.deleteStmt.run(docId);
            this.db.prepare('DELETE FROM ngram_docs WHERE doc_id = ?').run(docId);
        })();
        this.storageDirty = true;
    }
    /** Check if a document is already indexed */
    isIndexed(docId) {
        return !!this.docExistsStmt.get(docId);
    }
    /** Get count of indexed documents */
    getDocCount() {
        const row = this.db.prepare('SELECT COUNT(*) as cnt FROM ngram_docs').get();
        return row.cnt;
    }
    // ── Footprint accessors ─────────────────────────────────────────────────
    /** Path to the underlying SQLite database file. */
    getDbPath() {
        return this.dbPath;
    }
    /** Bytes occupied by the .db file alone. */
    getDbBytes() {
        return safeStatSize(this.dbPath);
    }
    /** Bytes occupied by the .db-wal file. */
    getWalBytes() {
        return safeStatSize(getNgramWalPath(this.dbPath));
    }
    /** Bytes occupied by the .db-shm file. */
    getShmBytes() {
        return safeStatSize(getNgramShmPath(this.dbPath));
    }
    /** Total SQLite footprint = db + wal + shm. Use this for quota checks. */
    getSqliteFootprintBytes() {
        return getNgramFootprintBytes(this.dbPath);
    }
    // ── Search ──────────────────────────────────────────────────────────────
    /**
     * Search for documents matching a text query.
     * Returns candidate document IDs ranked by trigram match count.
     *
     * The caller should do full text matching on these candidates —
     * this is a pre-filter, not a final answer.
     */
    search(query, limit = 50) {
        const queryTrigrams = extractTrigrams(query);
        if (queryTrigrams.length === 0)
            return [];
        // Deduplicate query trigrams by hash
        const uniqueHashes = new Map();
        for (const entry of queryTrigrams) {
            if (!uniqueHashes.has(entry.hash)) {
                uniqueHashes.set(entry.hash, entry);
            }
        }
        // Skip hashes we know aren't in the index.
        // If knownHashes is empty (warmup was deferred or skipped for a large DB),
        // fall through to the DB lookup for all unique query hashes rather than
        // returning [] — correctness beats the fast-path filter in that case.
        const allEntries = [...uniqueHashes.values()];
        const relevantEntries = this.knownHashes.size > 0
            ? allEntries.filter((entry) => this.knownHashes.has(entry.hash))
            : allEntries;
        if (relevantEntries.length === 0)
            return [];
        // Look up posting lists and score by match count
        const docScores = new Map();
        for (const entry of relevantEntries) {
            const postings = this.lookupStmt.all(entry.hash);
            const queryLocMask = buildLocMask(entry.position);
            const queryNextMask = buildNextMask(entry.nextChar);
            for (const posting of postings) {
                // Probabilistic mask filtering:
                // Check if position mask overlaps (trigram appears at compatible position)
                // and next-char bloom filter overlaps
                const locMatch = (posting.locMask & queryLocMask) !== 0 || posting.locMask === 0xFF;
                const nextMatch = (posting.nextMask & queryNextMask) !== 0 || posting.nextMask === 0xFF;
                if (locMatch && nextMatch) {
                    docScores.set(posting.docId, (docScores.get(posting.docId) ?? 0) + 1);
                }
            }
        }
        // Rank by number of matching trigrams (more = more selective match)
        return [...docScores.entries()]
            .map(([docId, score]) => ({ docId, score }))
            .sort((a, b) => b.score - a.score)
            .slice(0, limit);
    }
    /**
     * Search with a minimum trigram match threshold.
     * Only returns documents matching at least `minMatches` trigrams.
     */
    searchStrict(query, minMatches = 2, limit = 50) {
        return this.search(query, limit)
            .filter((r) => r.score >= minMatches)
            .map((r) => r.docId);
    }
    /**
     * Fast existence check: does any document contain this exact substring?
     * Uses trigram intersection for speed, then verifies against doc text
     * that the caller must provide.
     */
    candidatesForSubstring(substring, limit = 100) {
        if (substring.length < 3) {
            // Too short for trigrams — return all docs (caller must filter)
            return this.db.prepare('SELECT doc_id FROM ngram_docs LIMIT ?').all(limit)
                .map((r) => r.doc_id);
        }
        const results = this.search(substring, limit);
        // For substring search, require at least half the trigrams to match
        const minMatches = Math.max(1, Math.floor((substring.length - 2) / 2));
        return results
            .filter((r) => r.score >= minMatches)
            .map((r) => r.docId);
    }
    // ── Search with analytics ──────────────────────────────────────────────
    /**
     * Search with token savings tracking. Returns results + estimated tokens saved.
     * Use this method from MCP tools to get visible savings metrics.
     */
    searchWithStats(query, limit = 20) {
        const results = this.search(query, limit);
        // Estimate tokens saved: for each result, get doc text_length and compute tokens
        let totalCandidateTokens = 0;
        for (const result of results) {
            const doc = this.db.prepare('SELECT text_length FROM ngram_docs WHERE doc_id = ?').get(result.docId);
            if (doc) {
                totalCandidateTokens += Math.ceil(doc.text_length / 4); // ~4 chars/token
            }
        }
        // Tokens saved = full file tokens - query tokens (query is much smaller)
        const queryTokens = Math.ceil(query.length / 4);
        const tokensSaved = Math.max(0, totalCandidateTokens - queryTokens);
        // Record stats
        this._searchStats.totalQueries++;
        this._searchStats.totalTokensSaved += tokensSaved;
        this._searchStats.totalResultsReturned += results.length;
        this._searchStats.recentQueries.push({
            query: query.slice(0, 100),
            resultCount: results.length,
            tokensSaved,
            timestamp: Date.now(),
        });
        // Keep only last 50 recent queries
        if (this._searchStats.recentQueries.length > 50) {
            this._searchStats.recentQueries = this._searchStats.recentQueries.slice(-50);
        }
        return { results, tokensSaved, totalCandidateTokens };
    }
    /**
     * Sparse n-gram search: uses variable-length segments weighted by CRC32
     * character-pair rarity for higher selectivity on rare patterns.
     */
    searchSparse(query, limit = 20) {
        const sparseNgrams = extractSparseNgrams(query);
        if (sparseNgrams.length === 0)
            return this.search(query, limit);
        // Use first trigram of each sparse n-gram for index lookup
        const docScores = new Map();
        for (const entry of sparseNgrams) {
            if (!this.knownHashes.has(entry.hash))
                continue;
            const postings = this.lookupStmt.all(entry.hash);
            for (const posting of postings) {
                docScores.set(posting.docId, (docScores.get(posting.docId) ?? 0) + 1);
            }
        }
        return [...docScores.entries()]
            .map(([docId, score]) => ({ docId, score }))
            .sort((a, b) => b.score - a.score)
            .slice(0, limit);
    }
    /** Get accumulated search statistics */
    getSearchStats() {
        return { ...this._searchStats, recentQueries: [...this._searchStats.recentQueries] };
    }
    // ── Bulk Operations ─────────────────────────────────────────────────────
    /** Batch index multiple documents efficiently */
    addDocuments(docs) {
        let indexed = 0;
        const batchSize = 100;
        for (let i = 0; i < docs.length; i += batchSize) {
            const batch = docs.slice(i, i + batchSize);
            this.db.transaction(() => {
                for (const doc of batch) {
                    this.addDocument(doc.id, doc.text);
                    indexed++;
                }
            })();
        }
        return indexed;
    }
    /** Rebuild the entire index from scratch */
    rebuild(docs) {
        this.db.exec('DELETE FROM ngram_postings');
        this.db.exec('DELETE FROM ngram_docs');
        this.knownHashes.clear();
        this.storageDirty = true;
        return this.addDocuments(docs);
    }
    optimizeStorage(force = false) {
        try {
            this.db.exec('PRAGMA optimize');
        }
        catch {
            // Best effort optimize.
        }
        if (!force && !this.storageDirty) {
            return;
        }
        try {
            this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
        }
        catch {
            // Best effort checkpoint before vacuum.
        }
        this.db.exec('VACUUM');
        this.storageDirty = false;
    }
    /**
     * Operator-focused maintenance for the on-disk ngram DB.
     * - Bounds runaway DB growth via rotation, counting the
     *   full SQLite footprint (db + wal + shm) so a runaway WAL triggers it.
     * - Vacuums only when safe (<= vacuumMaxBytes) and either forced or dirty.
     */
    maintainBounded(options = {}) {
        const dbBytes = safeStatSize(this.dbPath);
        const footprint = this.getSqliteFootprintBytes();
        const rotateBytes = readEnvBytes('NEXUS_NGRAM_ROTATE_BYTES', NGRAM_DEFAULT_ROTATE_BYTES);
        const vacuumMaxBytes = readEnvBytes('NEXUS_NGRAM_VACUUM_MAX_BYTES', 256 * 1024 * 1024);
        if (footprint >= rotateBytes && footprint > 0) {
            const archiveEnabled = process.env.NEXUS_NGRAM_ARCHIVE_OVERSIZE === '1';
            const rotatedPath = `${this.dbPath}.oversize.${Date.now()}`;
            const removeSibling = (suffix) => {
                try {
                    fs.rmSync(`${this.dbPath}${suffix}`, { force: true });
                }
                catch { /* best effort */ }
            };
            try {
                try {
                    this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
                }
                catch { /* best effort */ }
                try {
                    this.db.close();
                }
                catch { /* ignore */ }
                if (archiveEnabled) {
                    fs.renameSync(this.dbPath, rotatedPath);
                }
                else {
                    fs.rmSync(this.dbPath, { force: true });
                }
                removeSibling('-wal');
                removeSibling('-shm');
            }
            catch (err) {
                // Re-open if close succeeded but rename failed
                try {
                    this.db = new Database(this.dbPath);
                    this.initSchema();
                    this.prepareStatements();
                }
                catch { /* ignore */ }
                return {
                    action: 'none',
                    dbPath: this.dbPath,
                    sizeBytes: footprint,
                    reason: `rotate failed: ${String(err?.message ?? err)}`,
                };
            }
            // Recreate a fresh DB
            this.knownHashes.clear();
            this.storageDirty = false;
            this.writesSinceCheckpoint = 0;
            this.lastCheckpointAt = Date.now();
            this.quotaSkipNoticeShown = false;
            this.db = new Database(this.dbPath);
            this.initSchema();
            this.prepareStatements();
            this.warmHashSet();
            return {
                action: 'rotated',
                dbPath: this.dbPath,
                sizeBytes: footprint,
                previousPath: archiveEnabled ? rotatedPath : undefined,
                reason: `footprint exceeded rotate threshold (db=${dbBytes}, footprint=${footprint}, threshold=${rotateBytes})`,
            };
        }
        if (dbBytes > 0 && dbBytes <= vacuumMaxBytes && (options.force || this.storageDirty)) {
            this.optimizeStorage(true);
            return {
                action: 'vacuum',
                dbPath: this.dbPath,
                sizeBytes: dbBytes,
                reason: options.force ? 'forced vacuum' : 'storageDirty vacuum',
            };
        }
        return {
            action: 'none',
            dbPath: this.dbPath,
            sizeBytes: footprint,
            reason: dbBytes > vacuumMaxBytes
                ? `skip vacuum: db too large (${dbBytes} > ${vacuumMaxBytes})`
                : 'no maintenance needed',
        };
    }
    // ── Stats ───────────────────────────────────────────────────────────────
    getStats() {
        const docCount = this.getDocCount();
        const postingRow = this.db.prepare('SELECT COUNT(*) as cnt FROM ngram_postings').get();
        const trigramRow = this.db.prepare('SELECT COUNT(DISTINCT ngram_hash) as cnt FROM ngram_postings').get();
        let dbSizeBytes = 0;
        try {
            dbSizeBytes = fs.statSync(this.dbPath).size;
        }
        catch { /* ignore */ }
        return {
            docCount,
            postingCount: postingRow.cnt,
            uniqueTrigramCount: trigramRow.cnt,
            dbSizeBytes,
        };
    }
    /** Close the database connection.
     *
     *  Always truncates the WAL so the .db-wal sibling can never outgrow the
     *  configured cap on shutdown — the failure mode that produced the 84GB
     *  WAL on a user machine. VACUUM is only run when the operator has both a
     *  small DB and explicitly opts in via NEXUS_NGRAM_VACUUM_ON_CLOSE=1, since
     *  blind VACUUM on multi-GB DBs blocks shutdown indefinitely. */
    close() {
        try {
            this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
        }
        catch {
            // best effort — never block shutdown on checkpoint failure
        }
        if (process.env.NEXUS_NGRAM_VACUUM_ON_CLOSE === '1') {
            const vacuumMaxBytes = readEnvBytes('NEXUS_NGRAM_VACUUM_MAX_BYTES', 256 * 1024 * 1024);
            const dbBytes = safeStatSize(this.dbPath);
            if (this.storageDirty && dbBytes > 0 && dbBytes <= vacuumMaxBytes) {
                try {
                    this.db.exec('VACUUM');
                    this.storageDirty = false;
                }
                catch { /* best effort */ }
            }
        }
        this.db.close();
    }
}
// ─────────────────────────────────────────────────────────────────────────────
// Singleton factory
// ─────────────────────────────────────────────────────────────────────────────
const _sharedInstances = new Map();
/** Get or create the shared NgramIndex instance */
export function getSharedNgramIndex(dbPath) {
    const key = path.resolve(dbPath ?? path.join(resolveNexusStateDir(), 'ngram-index.db'));
    let instance = _sharedInstances.get(key);
    if (!instance) {
        instance = new NgramIndex(key);
        _sharedInstances.set(key, instance);
    }
    return instance;
}
