/**
 * VectorService
 *
 * Manages semantic and full-text search using LanceDB.
 *
 * Tier 1 — FTS only (no API key required):
 *   All entities are indexed in LanceDB with a BM25 full-text search index on
 *   the `text` column. This is a significant upgrade over substring matching.
 *
 * Tier 2 — Hybrid semantic search (requires Voyage AI API key):
 *   When a voyage_api_key is configured, embeddings are generated via Voyage AI
 *   (voyage-3, 1024 dimensions) and stored alongside the FTS index. Searches
 *   use LanceDB hybrid mode (vector + FTS re-ranked).
 *
 * Storage: ~/.studiograph/vector/ (configurable via user.json vector_db_path)
 * Last-index timestamps: ~/.studiograph/vector/last-indexed.json
 */
import { join } from 'path';
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
import * as lancedb from '@lancedb/lancedb';
import { embeddedAssetIds } from '../core/embeds.js';
import { isCaptionableImage } from './assets/base.js';
import { tryResolve } from '../core/secrets/read-flip.js';
const VOYAGE_API_URL = 'https://api.voyageai.com/v1/embeddings';
const VOYAGE_MODEL = 'voyage-3';
const VECTOR_DIM = 1024;
const TABLE_NAME = 'entities';
/**
 * Dedicated LanceDB table for an asset's MUTABLE index state (decision A1).
 *
 * Why a separate table, not columns on `entities` and not asset frontmatter:
 *  - Frontmatter is immutable + git-committed; writing `index_status` there
 *    would fire one async git commit per asset and defeat the import batching.
 *    (The immutable `indexable` flag DOES live in frontmatter — see P3a.)
 *  - New columns on the shared `entities` table force a schema migration on
 *    every existing row. A side table is additive and touches only assets.
 */
const INDEX_STATE_TABLE = 'asset_index_state';
/**
 * Derivation-logic version. BUMP this when caption/extraction output changes so
 * the boot sweep RE-derives every asset (Codex migration rule: a row at a stale
 * version is re-derived, never treated as "done"). Legacy assets have NO row at
 * all — also re-derived, never skipped.
 */
export const ASSET_INDEX_VERSION = 1;
export class VectorService {
    dbPath;
    /**
     * Legacy fallback key passed via constructor for tests or for early-
     * boot code paths that run before the secrets store is initialized.
     * Live calls prefer `tryResolve('voyage.api_key')` so values written
     * to the store via the UI / CLI take effect without a restart.
     */
    fallbackVoyageApiKey;
    db;
    table;
    /** Lazily-opened side table for mutable asset index state (A1). */
    stateTable;
    stateInitialized = false;
    lastIndexedPath;
    initialized = false;
    // ── Single-flight guards (one in-flight init per resource) ──
    // `init()` and `initStateTable()` both open the LanceDB connection and can be
    // invoked concurrently (boot incremental reindex via upsert() races the
    // fire-and-forget asset sweep via getAllIndexStates()). Without
    // single-flighting, both would pass their boolean guard before either await
    // resolved, double-connect, and double-`createTable` → "table already exists"
    // / CommitConflict. Each promise is created once and awaited by all callers.
    dbPromise;
    initPromise;
    stateInitPromise;
    // ── Write serialization ──
    // LanceDB uses optimistic concurrency: two `mergeInsert(...).execute()` (or
    // delete) calls racing on the SAME table throw a CommitConflict and one loses.
    // This singleton VectorService is shared by foreground entity writes
    // (graph.ts fire-and-forget upsert/delete) AND the live asset-index drain
    // (re-upsert after writing a caption sidecar). Without serialization those two
    // sources can collide on the `entities` table — and the drain would record the
    // loser as a terminal `failed` index state. `withWrite()` chains every mutating
    // op onto a single per-instance promise so they run one at a time, in order.
    writeChain = Promise.resolve();
    /**
     * Fired exactly once per asset, when `ensureInitialAssetIndexState` seeds a
     * fresh `pending` row — i.e. an indexable IMAGE assets.db row that needs
     * captioning. The asset-creation paths (graph.uploadAsset for import +
     * manager/editor uploads, asset-finalize-service for presign) call
     * ensureInitialAssetIndexState after writing the assets.db row, so the live
     * caption drain registers here instead of being wired at each call site. It
     * fires only for images and only on the first seed (a re-seed early-returns),
     * so it can't loop the drain.
     */
    onAssetPendingCb;
    /**
     * Workspace-global asset ownership index (assets.db). Injected by the
     * WorkspaceManager so `upsert` can resolve the captions of images an entry
     * embeds (`resolveEmbeddedCaptions`) and fold them into the entry's search
     * text. Optional — tests / early-boot paths run without it (no caption fold).
     */
    assetIndex;
    // ── Operator-visible health (surfaced via /api/health/dirty) ──
    // Tracked here, not at the call sites, because every fire-and-forget
    // upsert/delete in graph.ts catches its own promise to avoid blocking
    // writes — the operator needs a single rolled-up signal instead of
    // grepping logs for `[vector] index failed`.
    failureCount = 0;
    inFlight = 0;
    lastFailureAt = null;
    lastFailure = null;
    constructor(dbPath, voyageApiKey) {
        this.dbPath = dbPath;
        this.fallbackVoyageApiKey = voyageApiKey;
        this.lastIndexedPath = join(dbPath, 'last-indexed.json');
    }
    /**
     * Resolve the Voyage API key fresh on each call: store > constructor
     * fallback. Lets the operator flip the key via UI / CLI without a
     * server restart.
     */
    get voyageApiKey() {
        const fromStore = tryResolve('voyage.api_key');
        return fromStore ?? this.fallbackVoyageApiKey;
    }
    /** Begin tracking an in-flight op. Pair with `recordSuccess`/`recordFailure`. */
    beginOp() { this.inFlight++; }
    recordSuccess() {
        if (this.inFlight > 0)
            this.inFlight--;
    }
    recordFailure(op, entity, err) {
        if (this.inFlight > 0)
            this.inFlight--;
        this.failureCount++;
        this.lastFailureAt = new Date().toISOString();
        this.lastFailure = {
            op,
            entity,
            error: err instanceof Error ? err.message : String(err),
        };
    }
    /**
     * Snapshot for /api/health. Healthy = the last op succeeded (or none have
     * run yet) AND there is no abnormal in-flight backlog. The failure count
     * is informational; a single transient Voyage 503 shouldn't permanently
     * mark the service unhealthy, but operators can see the trend.
     */
    getHealth() {
        return {
            healthy: this.lastFailure === null,
            failureCount: this.failureCount,
            inFlight: this.inFlight,
            lastFailureAt: this.lastFailureAt,
            lastFailure: this.lastFailure,
            embeddingsEnabled: !!this.voyageApiKey,
        };
    }
    /** Inject the workspace-global asset ownership index (assets.db). Lets
     *  `upsert` fold embedded-image captions into an entry's search text. */
    setAssetIndex(index) {
        this.assetIndex = index;
    }
    /**
     * The captions of every immutable-keyed image an entry embeds, joined for
     * folding into the entry's FTS/embedding text (asset-as-embed, decision D2).
     *
     * The caption + OCR text is derived once per image and stored in
     * `assets.db.caption`, keyed by the immutable assetId (NOT in a sidecar or
     * the body). An entry becomes findable by an embedded image's visual content
     * by reading those captions back here — for the `material` body embed, the
     * `reference` thumbnail, a note body image, etc. Each captioned image's text
     * is read from the single assets.db row, so the same image embedded by N
     * entries enriches all N without duplicating the stored caption.
     *
     * Forward-only (entry → the assets IT embeds) — no reverse index, no
     * cross-repo ownership claim. Best-effort: returns undefined without an index
     * or when nothing embedded is captioned.
     */
    resolveEmbeddedCaptions(entity) {
        const index = this.assetIndex;
        if (!index)
            return undefined;
        // `embeddedAssetIds` (core/embeds.ts) is the SHARED definition of "which
        // assets does this entry embed" — body + string frontmatter, immutable-key
        // urls only — also used by the sweep's embedder re-index, so the two never
        // disagree on coverage. It applies a cheap substring pre-check, so a no-asset
        // entry costs no regex.
        const ids = embeddedAssetIds(entity.document?.content ?? '', entity.data);
        if (ids.size === 0)
            return undefined;
        const captions = [];
        for (const id of ids) {
            try {
                const caption = index.get(id)?.caption?.trim();
                if (caption)
                    captions.push(caption);
            }
            catch { /* best-effort: a lookup failure must not fail the index */ }
        }
        return captions.length ? captions.join('\n') : undefined;
    }
    /**
     * Open/create the LanceDB table and ensure the FTS index exists.
     * Call once before using the service.
     */
    async init() {
        if (this.initialized)
            return;
        // Single-flight: concurrent callers share one in-flight init.
        if (!this.initPromise)
            this.initPromise = this.doInit();
        await this.initPromise;
    }
    /**
     * Run a mutating LanceDB op under the per-instance write lock so concurrent
     * writers (foreground entity writes vs. the asset-index drain) can't collide
     * on a shared table's optimistic-concurrency commit. Ops run strictly in the
     * order `withWrite` is called; a throwing op rejects to ITS caller without
     * poisoning the chain for the next writer.
     */
    withWrite(fn) {
        const run = this.writeChain.then(fn, fn);
        // Keep the chain alive regardless of this op's outcome (swallow on the
        // chain copy only; the real result/throw still goes to the caller).
        this.writeChain = run.then(() => undefined, () => undefined);
        return run;
    }
    /**
     * Open (once) the shared LanceDB connection. Both `init()` and
     * `initStateTable()` go through this so they never race to assign `this.db`.
     */
    async ensureDb() {
        if (this.db)
            return this.db;
        if (!this.dbPromise) {
            mkdirSync(this.dbPath, { recursive: true });
            this.dbPromise = lancedb.connect(this.dbPath);
        }
        this.db = await this.dbPromise;
        return this.db;
    }
    async doInit() {
        const db = await this.ensureDb();
        const tableNames = await db.tableNames();
        if (tableNames.includes(TABLE_NAME)) {
            this.table = await db.openTable(TABLE_NAME);
        }
        else {
            // Create table with a seed record (LanceDB infers schema from first batch)
            const seed = {
                id: '__seed__',
                repo: '',
                entity_type: '',
                entity_id: '',
                text: 'seed',
                vector: new Array(VECTOR_DIM).fill(0),
                updated_at: new Date().toISOString(),
            };
            this.table = await db.createTable(TABLE_NAME, [seed]);
            // Create FTS index on text column
            await this.table.createIndex('text', {
                config: lancedb.Index.fts(),
            });
            // Delete the seed record
            await this.table.delete(`id = '__seed__'`);
        }
        this.initialized = true;
    }
    // ─── Asset index state (mutable, LanceDB-only; keyed by assetId) ─────────────
    /**
     * Open/create the side table for asset index state. Independent of `init()`
     * so state-only callers (and tests) don't pay for the entities table. Lazy
     * because most workspaces never need it until the first asset is indexed.
     */
    async initStateTable() {
        if (this.stateInitialized)
            return;
        // Single-flight: concurrent state readers/writers share one in-flight init,
        // so two callers can't both take the createTable branch (which would throw
        // "table already exists" on the loser).
        if (!this.stateInitPromise)
            this.stateInitPromise = this.doInitStateTable();
        await this.stateInitPromise;
    }
    async doInitStateTable() {
        const db = await this.ensureDb();
        const names = await db.tableNames();
        if (names.includes(INDEX_STATE_TABLE)) {
            this.stateTable = await db.openTable(INDEX_STATE_TABLE);
        }
        else {
            const seed = {
                id: '__seed__', repo: '', entity_id: '', index_status: 'pending',
                indexed_at: '', index_version: 0, reason: '',
            };
            this.stateTable = await db.createTable(INDEX_STATE_TABLE, [seed]);
            await this.stateTable.delete(`id = '__seed__'`);
        }
        this.stateInitialized = true;
    }
    toState(r) {
        return {
            assetId: r.entity_id, // legacy column name; holds the immutable UUID
            repo: r.repo,
            status: r.index_status,
            indexedAt: r.indexed_at,
            version: r.index_version,
            reason: r.reason,
        };
    }
    /**
     * Read the `text` actually indexed for one entity (the value `buildEntityText`
     * produced — markdown body plus, for assets, the extracted sidecar). Null when
     * no row exists. This is the searchable content of record, independent of FTS
     * index freshness, so it's the deterministic way to assert "the asset's body
     * reached the index" (e.g. the docx stub-upsert-race regression test).
     */
    async getIndexedText(repo, entityType, entityId) {
        await this.init();
        const id = `${repo}/${entityType}/${entityId}`.replace(/'/g, "''");
        const rows = await this.table.query().where(`id = '${id}'`).limit(1).toArray();
        const row = rows.find((r) => r.id !== '__seed__');
        return row ? row.text : null;
    }
    /** Read one asset's index state, or null when no row exists (never-derived). */
    async getIndexState(assetId) {
        await this.initStateTable();
        const id = assetId.replace(/'/g, "''");
        const rows = await this.stateTable.query().where(`id = '${id}'`).limit(1).toArray();
        const row = rows.find((r) => r.id !== '__seed__');
        return row ? this.toState(row) : null;
    }
    /**
     * Load every index-state row into a Map keyed by assetId. The boot sweep uses
     * this to decide, per image asset, whether a row is missing (never-derived),
     * stale (old version), or unfinished (pending) — without re-deriving the
     * whole set (PERF1).
     */
    async getAllIndexStates() {
        await this.initStateTable();
        const rows = await this.stateTable.query().toArray();
        const map = new Map();
        for (const r of rows) {
            if (r.id === '__seed__')
                continue;
            // Key by the authoritative PK `id` (== assetId for current rows), matching
            // getIndexState/setIndexState. Legacy entity-keyed rows from the old
            // pipeline have `id = '<repo>/<entityId>'`, which can never equal a UUID
            // assetId, so they stay inert here (the sweep looks up by assetId) rather
            // than being mistaken for a current row via their bare `entity_id`.
            map.set(r.id, this.toState(r));
        }
        return map;
    }
    /** Upsert an asset's index state. Stamps `indexed_at` + current version unless overridden. */
    async setIndexState(assetId, status, opts = {}) {
        await this.initStateTable();
        const record = {
            id: assetId,
            repo: opts.repo ?? '',
            entity_id: assetId,
            index_status: status,
            indexed_at: opts.indexedAt ?? new Date().toISOString(),
            index_version: opts.version ?? ASSET_INDEX_VERSION,
            reason: opts.reason ?? '',
        };
        await this.withWrite(() => this.stateTable
            .mergeInsert('id')
            .whenMatchedUpdateAll()
            .whenNotMatchedInsertAll()
            .execute([record]));
    }
    /** Drop an asset's index-state row (e.g. when the asset is deleted). */
    async deleteIndexState(assetId) {
        await this.initStateTable();
        const id = assetId.replace(/'/g, "''");
        await this.withWrite(() => this.stateTable.delete(`id = '${id}'`));
    }
    /**
     * Register the single sink for "a new indexable image needs captioning."
     * The live caption drain (AssetIndexQueue) registers here; the chokepoint
     * fires from `ensureInitialAssetIndexState`, covering every image-upload path
     * at once. Replaces per-call-site enqueue wiring.
     */
    setOnAssetPending(cb) {
        this.onAssetPendingCb = cb;
    }
    /**
     * Seed the initial index state for a freshly-stored IMAGE asset, IF it has no
     * row yet, and notify the live caption drain. Called by the asset-creation
     * paths (graph.uploadAsset, asset-finalize-service) after the assets.db row
     * is written — keyed by the immutable assetId, never an entity.
     *
     * State *transitions* (full/failed) are owned by the caption sweep; this only
     * establishes the `pending` starting point + fires the drain. Only raster
     * images are captioned: SVG stays transitional (sanitize-html not wired);
     * documents/opaque binaries carry no caption. A re-seed (row already exists)
     * early-returns so the drain can't loop. Best-effort: a state-write failure
     * must not fail the upload.
     */
    async ensureInitialAssetIndexState(assetId, opts) {
        if (!isCaptionableImage(opts.mediaType, opts.filename))
            return;
        try {
            const existing = await this.getIndexState(assetId);
            if (existing)
                return;
            await this.setIndexState(assetId, 'pending', { repo: opts.repo });
            try {
                this.onAssetPendingCb?.(assetId);
            }
            catch { /* non-fatal */ }
        }
        catch (err) {
            // eslint-disable-next-line no-console
            console.warn(`[vector] asset index-state seed failed for ${assetId}: ${err instanceof Error ? err.message : err}`);
        }
    }
    /**
     * Index or re-index a single entity.
     * Safe to call on create, update, or as part of bulk indexing.
     */
    async upsert(repoName, entity) {
        await this.init();
        const id = `${repoName}/${entity.entityType}/${entity.id}`;
        const text = buildEntityText(entity, this.resolveEmbeddedCaptions(entity));
        const vector = this.voyageApiKey
            ? await this.generateEmbedding(text)
            : null;
        const record = {
            id,
            repo: repoName,
            entity_type: entity.entityType,
            entity_id: entity.id,
            text,
            // Only include vector when we actually have one — avoids LanceDB schema inference issues
            vector: vector ?? new Array(VECTOR_DIM).fill(0),
            updated_at: new Date().toISOString(),
        };
        await this.withWrite(() => this.table
            .mergeInsert('id')
            .whenMatchedUpdateAll()
            .whenNotMatchedInsertAll()
            .execute([record]));
    }
    /**
     * Bulk upsert — used by `studiograph index`. Batches Voyage API calls.
     */
    async upsertMany(repoName, entities, opts = {}) {
        await this.init();
        if (entities.length === 0)
            return;
        const texts = entities.map(e => buildEntityText(e, this.resolveEmbeddedCaptions(e)));
        // Batch embeddings (Voyage supports up to 128 per request)
        let vectors = new Array(entities.length).fill(null);
        if (this.voyageApiKey) {
            vectors = await this.generateEmbeddingsBatch(texts, opts.onProgress);
        }
        else {
            // No embedding step — report the whole batch as "done" once so
            // progress UIs that wait on at least one callback don't sit at 0%.
            opts.onProgress?.(entities.length, entities.length);
        }
        const records = entities.map((entity, i) => ({
            id: `${repoName}/${entity.entityType}/${entity.id}`,
            repo: repoName,
            entity_type: entity.entityType,
            entity_id: entity.id,
            text: texts[i],
            vector: vectors[i] ?? new Array(VECTOR_DIM).fill(0),
            updated_at: new Date().toISOString(),
        }));
        await this.withWrite(() => this.table
            .mergeInsert('id')
            .whenMatchedUpdateAll()
            .whenNotMatchedInsertAll()
            .execute(records));
    }
    /**
     * Remove an entity from the index.
     */
    async delete(repoName, entityType, entityId) {
        await this.init();
        const id = `${repoName}/${entityType}/${entityId}`;
        await this.withWrite(() => this.table.delete(`id = '${id.replace(/'/g, "''")}'`));
        // No asset index-state reap here: under asset-as-embed the caption state is
        // keyed by the immutable assetId and survives deletion of any entry that
        // merely embeds the asset. The state row is reaped only when the asset
        // itself is deleted (graph.deleteAssetById → vectorService.deleteIndexState).
    }
    /**
     * Remove an entity by `(repo, id)` alone, without needing its type.
     *
     * Used by the serve-boot incremental indexer: when git diff reports a
     * deletion, the file is gone from disk so its type can't be resolved
     * from frontmatter. The vector store has a discrete `entity_id` column
     * so we can match on `repo + entity_id` without falling back to a LIKE
     * pattern on the synthetic id.
     *
     * Safe across type renames: a single id that legitimately had two type
     * rows (e.g. type changed mid-edit and both old + new still exist) gets
     * both wiped — which is the right behavior for incremental rebuild
     * since the next upsert recreates the current type's row.
     */
    async deleteByRepoAndId(repoName, entityId) {
        await this.init();
        const r = repoName.replace(/'/g, "''");
        const e = entityId.replace(/'/g, "''");
        await this.withWrite(() => this.table.delete(`repo = '${r}' AND entity_id = '${e}'`));
        // No asset index-state reap here either — see delete(): asset state is
        // assetId-keyed and decoupled from the entry's lifecycle.
    }
    /**
     * Best-effort `deleteIndexState` for the asset-delete path: the blob/index
     * delete is the authoritative op, so a failure to reap the state row must not
     * turn a successful asset delete into a thrown error. Called by
     * graph.deleteAssetById.
     */
    async deleteIndexStateSafe(assetId) {
        try {
            await this.deleteIndexState(assetId);
        }
        catch (err) {
            // eslint-disable-next-line no-console
            console.warn(`[vector] index-state cleanup failed for ${assetId}: ${err instanceof Error ? err.message : err}`);
        }
    }
    /**
     * Search the index.
     * Uses hybrid search (vector + FTS) when Voyage key is configured, FTS only otherwise.
     */
    async search(query, options = {}) {
        await this.init();
        const limit = options.limit ?? 10;
        try {
            let rows;
            // Repo scoping is a PRE-filter pushed into the query, not a post-filter on
            // the result page. Filtering after `.limit()` silently drops a scoped
            // repo's matches whenever the global top-K is dominated by other repos —
            // e.g. a `repos:['studiograph']` search returning nothing while the same
            // query unscoped ranks a studiograph doc #1. `where` applies BEFORE the
            // limit so the K results are already within scope.
            const repoWhere = options.repos && options.repos.length > 0
                ? options.repos.map(r => `repo = '${r.replace(/'/g, "''")}'`).join(' OR ')
                : null;
            if (this.voyageApiKey) {
                // Tier 2: hybrid search
                const queryVector = await this.generateEmbedding(query);
                let q = this.table.search(queryVector);
                if (repoWhere)
                    q = q.where(repoWhere);
                rows = await q
                    .limit(limit * 2) // fetch extra for re-ranking
                    .toArray();
            }
            else {
                // Tier 1: FTS only
                let q = this.table.search(query, 'text');
                if (repoWhere)
                    q = q.where(repoWhere);
                rows = await q
                    .limit(limit)
                    .toArray();
            }
            const results = rows
                .filter((r) => r.id !== '__seed__' && r.entity_id !== '')
                .map((r, i) => ({
                repoName: r.repo,
                entityType: r.entity_type,
                entityId: r.entity_id,
                score: r._distance != null ? 1 / (1 + r._distance) : 1 - i * 0.01,
            }));
            return results.slice(0, limit);
        }
        catch {
            // FTS index may not be built yet — return empty
            return [];
        }
    }
    /**
     * Get the last time a repo's entities were indexed.
     */
    getLastIndexedTime(repoName) {
        if (!existsSync(this.lastIndexedPath))
            return undefined;
        try {
            const data = JSON.parse(readFileSync(this.lastIndexedPath, 'utf-8'));
            return data[repoName];
        }
        catch {
            return undefined;
        }
    }
    /**
     * Record the last index time for a repo.
     *
     * Production callers omit `isoTimestamp` and the current time is used.
     * Tests pass an explicit ISO timestamp so they can rewind the clock and
     * assert the git-log `--since` window picks up the right commits.
     */
    setLastIndexedTime(repoName, isoTimestamp) {
        mkdirSync(this.dbPath, { recursive: true });
        let data = {};
        if (existsSync(this.lastIndexedPath)) {
            try {
                data = JSON.parse(readFileSync(this.lastIndexedPath, 'utf-8'));
            }
            catch { /* ignore */ }
        }
        data[repoName] = isoTimestamp ?? new Date().toISOString();
        writeFileSync(this.lastIndexedPath, JSON.stringify(data, null, 2) + '\n');
    }
    /** Whether Voyage AI embeddings are enabled (Tier 2). */
    get hasEmbeddings() {
        return !!this.voyageApiKey;
    }
    // ─── Private: Voyage AI ──────────────────────────────────────────────────────
    async generateEmbedding(text) {
        const results = await this.generateEmbeddingsBatch([text]);
        return results[0] ?? new Array(VECTOR_DIM).fill(0);
    }
    async generateEmbeddingsBatch(texts, onProgress) {
        // Voyage supports up to 128 inputs per request
        const BATCH_SIZE = 128;
        const all = [];
        const total = texts.length;
        for (let i = 0; i < texts.length; i += BATCH_SIZE) {
            const batch = texts.slice(i, i + BATCH_SIZE);
            const response = await fetch(VOYAGE_API_URL, {
                method: 'POST',
                headers: {
                    'Authorization': `Bearer ${this.voyageApiKey}`,
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({ model: VOYAGE_MODEL, input: batch }),
            });
            if (!response.ok) {
                const body = await response.text();
                throw new Error(`Voyage AI error ${response.status}: ${body}`);
            }
            const json = await response.json();
            all.push(...json.data.map(d => d.embedding));
            onProgress?.(all.length, total);
        }
        return all;
    }
}
// ─── Standalone reindex ──────────────────────────────────────────────────────
/**
 * Index (or re-index) all entities across workspace repos into LanceDB.
 * Standalone function — does not require a WorkspaceManager instance.
 * Used by `join`, `commit`, and other CLI commands that land entities on disk.
 *
 * @returns Total number of entities indexed.
 */
export async function reindexWorkspace(workspacePath, repos, onProgress) {
    const { loadUserConfig } = await import('../core/user-config.js');
    const { BaseGraphManager } = await import('../core/graph.js');
    const { AssetIndexService } = await import('./assets/asset-index-service.js');
    const { homedir } = await import('os');
    const userConfig = loadUserConfig();
    const dbPath = userConfig.vector_db_path || join(homedir(), '.studiograph', 'vector');
    const vectorService = new VectorService(dbPath, userConfig.voyage_api_key);
    // Wire the assets.db handle so a cold reindex still folds embedded-image
    // captions into entry search text (otherwise it would overwrite the server's
    // caption-folded rows with caption-less ones).
    vectorService.setAssetIndex(new AssetIndexService(workspacePath));
    await vectorService.init();
    const log = onProgress ?? (() => { });
    let total = 0;
    for (const repo of repos) {
        const repoPath = join(workspacePath, repo.path);
        if (!existsSync(repoPath))
            continue;
        try {
            const graph = new BaseGraphManager({
                repoPath,
                repoName: repo.name,
                gitUser: { id: 'reindex', name: 'reindex', email: 'reindex@studiograph.local' },
            });
            const entities = graph.list();
            if (entities.length > 0) {
                await vectorService.upsertMany(repo.name, entities);
                total += entities.length;
            }
            vectorService.setLastIndexedTime(repo.name);
            log(`  Indexed ${repo.name}: ${entities.length} entities`);
        }
        catch (err) {
            log(`  Failed to index ${repo.name}: ${err instanceof Error ? err.message : 'unknown'}`);
        }
    }
    return total;
}
export async function incrementalReindexWorkspace(workspaceManager, repos, workspacePath, opts = {}) {
    const vectorService = workspaceManager.vectorService;
    if (!vectorService) {
        return { total: 0, fresh: false };
    }
    const log = opts.log ?? (() => { });
    const { execFileSync } = await import('child_process');
    let total = 0;
    let anyFreshRepo = false;
    const staleEntities = [];
    // Pre-pass: figure out the total first-time entity count so the
    // progress callback has a stable denominator. Cheap — `graph.list()`
    // doesn't re-read files, it returns the in-memory graph state the
    // workspace manager already maintains.
    let firstTimeTotal = 0;
    if (opts.onFirstTimeProgress) {
        for (const repo of repos) {
            if (vectorService.getLastIndexedTime(repo.name))
                continue;
            try {
                firstTimeTotal += workspaceManager.getGraph(repo.name).list().length;
            }
            catch {
                // tolerate per-repo failure; the real loop below will log it
            }
        }
    }
    let firstTimeDone = 0;
    for (const repo of repos) {
        const lastIndexed = vectorService.getLastIndexedTime(repo.name);
        // Per-repo first-time index. Catches three cases:
        //   1. Very first `serve` boot (every repo missing a timestamp).
        //   2. New folder added between serve boots (existing workspace +
        //      new repo with no timestamp). Pre-fix this was skipped here
        //      and never indexed unless the operator ran a full reindex.
        //   3. Vector DB rebuilt from scratch but workspace has existing
        //      folders.
        if (!lastIndexed) {
            try {
                const graph = workspaceManager.getGraph(repo.name);
                const entities = graph.list();
                if (entities.length > 0) {
                    await vectorService.upsertMany(repo.name, entities, {
                        onProgress: (batchDone) => {
                            opts.onFirstTimeProgress?.(firstTimeDone + batchDone, firstTimeTotal);
                        },
                    });
                    firstTimeDone += entities.length;
                    total += entities.length;
                }
                vectorService.setLastIndexedTime(repo.name);
                anyFreshRepo = true;
            }
            catch (err) {
                log(`failed to index ${repo.name}: ${err instanceof Error ? err.message : 'unknown'}`);
            }
            continue;
        }
        // Established repo — diff git log since the last index timestamp and
        // collect a list of stale (repo, id) tuples to upsert / delete below.
        const repoPath = join(workspacePath, repo.path);
        if (!existsSync(repoPath))
            continue;
        try {
            const output = execFileSync('git', ['log', `--since=${lastIndexed}`, '--name-status', '--format=', '--', '*.md'], { cwd: repoPath, encoding: 'utf-8', stdio: 'pipe' }).trim();
            if (!output)
                continue;
            for (const line of output.split('\n')) {
                // Accept any markdown path. Layouts in the wild:
                //   - flat:    `entity-id.md`
                //   - typed:   `<type>/entity-id.md`
                //   - nested:  `folder/sub/entity-id.md`
                // STU-131 normalized on flat for new entities; older typed layouts
                // and arbitrary nested folders still appear in mixed repos. Entity
                // type lives in frontmatter (resolved via graph.getById for
                // upserts; not needed for deletes since we now match by `(repo,
                // entity_id)` columns). Pre-fix this regex required exactly one
                // slash with a `\w+` type segment and missed all three layouts,
                // plus hyphenated types like `case-study`.
                const m = line.match(/^([AMD])\s+(.+\.md)$/);
                if (!m)
                    continue;
                const [, status, path] = m;
                // basename minus `.md`. Entity IDs are unique within a repo.
                const id = path.split('/').pop().replace(/\.md$/, '');
                if (!id)
                    continue;
                staleEntities.push({ repoName: repo.name, entityId: id, deleted: status === 'D' });
            }
        }
        catch (err) {
            log(`failed to diff ${repo.name}: ${err instanceof Error ? err.message : 'unknown'}`);
        }
    }
    if (staleEntities.length === 0) {
        // Nothing changed via git diff. If we did first-time work above, that
        // counts as a fresh run; otherwise this was a no-op incremental boot.
        return { total, fresh: anyFreshRepo };
    }
    // Dedupe: if an entity appears multiple times (e.g. M then D), the last
    // entry wins. Order matters — git log is newest-first by default, so we
    // reverse before reducing to get the newest state per (repo, id).
    const seen = new Map();
    for (const item of staleEntities.slice().reverse()) {
        seen.set(`${item.repoName}/${item.entityId}`, item);
    }
    for (const item of seen.values()) {
        try {
            if (item.deleted) {
                // Delete by (repo, id) — type isn't recoverable from a deleted file.
                // The new column-based matcher in vectorService handles it without
                // a frontmatter lookup.
                await vectorService.deleteByRepoAndId(item.repoName, item.entityId);
            }
            else {
                const graph = workspaceManager.getGraph(item.repoName);
                const entity = graph.getById(item.entityId);
                await vectorService.upsert(item.repoName, entity);
                total++;
            }
        }
        catch {
            // Entity may have been deleted then re-created; safe to skip.
        }
    }
    // Stamp every repo we touched. Empty-diff repos already had a timestamp
    // (we skipped the rebuild path) — re-stamping is cheap and keeps the
    // "since" window tight for the next boot.
    for (const repo of repos) {
        vectorService.setLastIndexedTime(repo.name);
    }
    return { total, fresh: anyFreshRepo };
}
// ─── Helpers ─────────────────────────────────────────────────────────────────
/**
 * Build a text representation of an entity for FTS/embedding.
 * Combines entity type, key frontmatter fields, and markdown body.
 *
 * `extraText` is appended after the body — used by VectorService to
 * fold PDF sidecar text into asset-entity embeddings. Kept as a plain
 * string parameter (not file IO) so this stays pure and unit-testable.
 *
 * Exported for testing.
 */
export function buildEntityText(entity, extraText) {
    const parts = [];
    // Entity type and ID as context
    parts.push(`${entity.entityType}: ${entity.data.name || entity.id}`);
    // Key frontmatter fields (skip system fields and empty values)
    const systemFields = new Set([
        'entity_type', 'entity_id', 'created_at', 'updated_at',
        'created_by', 'updated_by', 'visibility', 'restricted_to',
        'sensitive_fields',
    ]);
    for (const [key, value] of Object.entries(entity.data)) {
        if (systemFields.has(key) || value == null || value === '')
            continue;
        if (Array.isArray(value)) {
            if (value.length > 0)
                parts.push(`${key}: ${value.join(', ')}`);
        }
        else {
            parts.push(`${key}: ${String(value)}`);
        }
    }
    // Markdown body
    if (entity.document.content) {
        // Strip markdown syntax for cleaner FTS/embedding input
        const plainText = entity.document.content
            .replace(/^#{1,6}\s+/gm, '') // headings
            .replace(/\*\*(.+?)\*\*/g, '$1') // bold
            .replace(/\*(.+?)\*/g, '$1') // italic
            .replace(/`(.+?)`/g, '$1') // inline code
            .replace(/\[\[(.+?)\]\]/g, '$1') // wikilinks
            .trim();
        if (plainText)
            parts.push(plainText);
    }
    if (extraText) {
        const trimmed = extraText.trim();
        if (trimmed)
            parts.push(trimmed);
    }
    return parts.join('\n');
}
//# sourceMappingURL=vector-service.js.map