/**
 * VectorService
 *
 * Manages semantic and full-text search using LanceDB.
 *
 * Tier 1 — FTS only (no API key required):
 *   All entities are indexed in LanceDB with a BM25 full-text search index on
 *   the `text` column. This is a significant upgrade over substring matching.
 *
 * Tier 2 — Hybrid semantic search (requires Voyage AI API key):
 *   When a voyage_api_key is configured, embeddings are generated via Voyage AI
 *   (voyage-3, 1024 dimensions) and stored alongside the FTS index. Searches
 *   use LanceDB hybrid mode (vector + FTS re-ranked).
 *
 * Storage: ~/.studiograph/vector/ (configurable via user.json vector_db_path)
 * Last-index timestamps: ~/.studiograph/vector/last-indexed.json
 */
import { join } from 'path';
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
import * as lancedb from '@lancedb/lancedb';
import { tryResolve } from '../core/secrets/read-flip.js';
const VOYAGE_API_URL = 'https://api.voyageai.com/v1/embeddings';
const VOYAGE_MODEL = 'voyage-3';
const VECTOR_DIM = 1024;
const TABLE_NAME = 'entities';
export class VectorService {
    dbPath;
    /**
     * Legacy fallback key passed via constructor for tests or for early-
     * boot code paths that run before the secrets store is initialized.
     * Live calls prefer `tryResolve('voyage.api_key')` so values written
     * to the store via the UI / CLI take effect without a restart.
     */
    fallbackVoyageApiKey;
    workspaceRoot;
    db;
    table;
    lastIndexedPath;
    initialized = false;
    // ── Operator-visible health (surfaced via /api/health/dirty) ──
    // Tracked here, not at the call sites, because every fire-and-forget
    // upsert/delete in graph.ts catches its own promise to avoid blocking
    // writes — the operator needs a single rolled-up signal instead of
    // grepping logs for `[vector] index failed`.
    failureCount = 0;
    inFlight = 0;
    lastFailureAt = null;
    lastFailure = null;
    constructor(dbPath, voyageApiKey, workspaceRoot) {
        this.dbPath = dbPath;
        this.fallbackVoyageApiKey = voyageApiKey;
        this.workspaceRoot = workspaceRoot;
        this.lastIndexedPath = join(dbPath, 'last-indexed.json');
    }
    /**
     * Resolve the Voyage API key fresh on each call: store > constructor
     * fallback. Lets the operator flip the key via UI / CLI without a
     * server restart.
     */
    get voyageApiKey() {
        const fromStore = tryResolve('voyage.api_key');
        return fromStore ?? this.fallbackVoyageApiKey;
    }
    /** Begin tracking an in-flight op. Pair with `recordSuccess`/`recordFailure`. */
    beginOp() { this.inFlight++; }
    recordSuccess() {
        if (this.inFlight > 0)
            this.inFlight--;
    }
    recordFailure(op, entity, err) {
        if (this.inFlight > 0)
            this.inFlight--;
        this.failureCount++;
        this.lastFailureAt = new Date().toISOString();
        this.lastFailure = {
            op,
            entity,
            error: err instanceof Error ? err.message : String(err),
        };
    }
    /**
     * Snapshot for /api/health. Healthy = the last op succeeded (or none have
     * run yet) AND there is no abnormal in-flight backlog. The failure count
     * is informational; a single transient Voyage 503 shouldn't permanently
     * mark the service unhealthy, but operators can see the trend.
     */
    getHealth() {
        return {
            healthy: this.lastFailure === null,
            failureCount: this.failureCount,
            inFlight: this.inFlight,
            lastFailureAt: this.lastFailureAt,
            lastFailure: this.lastFailure,
            embeddingsEnabled: !!this.voyageApiKey,
        };
    }
    /**
     * For asset entities backed by a PDF, the binary's extracted text
     * lives in a sidecar at `<workspace>/.studiograph/assets-text/asset/<id>.txt`
     * (written at upload time by `uploadAsset`). Returns the contents so
     * `buildEntityText` can fold the PDF into the embedding/FTS input. The
     * markdown body of an asset entity is intentionally light — without
     * this, PDF assets would be effectively invisible to RAG.
     *
     * Best-effort: missing sidecar / read errors return null. Asset stays
     * indexed by frontmatter alone.
     */
    readPdfSidecar(entity) {
        if (!this.workspaceRoot)
            return null;
        if (entity.entityType !== 'asset')
            return null;
        if (entity.data.asset_kind !== 'pdf')
            return null;
        const path = join(this.workspaceRoot, '.studiograph', 'assets-text', 'asset', `${entity.id}.txt`);
        try {
            if (!existsSync(path))
                return null;
            const text = readFileSync(path, 'utf-8').trim();
            return text || null;
        }
        catch {
            return null;
        }
    }
    /**
     * Open/create the LanceDB table and ensure the FTS index exists.
     * Call once before using the service.
     */
    async init() {
        if (this.initialized)
            return;
        mkdirSync(this.dbPath, { recursive: true });
        this.db = await lancedb.connect(this.dbPath);
        const tableNames = await this.db.tableNames();
        if (tableNames.includes(TABLE_NAME)) {
            this.table = await this.db.openTable(TABLE_NAME);
        }
        else {
            // Create table with a seed record (LanceDB infers schema from first batch)
            const seed = {
                id: '__seed__',
                repo: '',
                entity_type: '',
                entity_id: '',
                text: 'seed',
                vector: new Array(VECTOR_DIM).fill(0),
                updated_at: new Date().toISOString(),
            };
            this.table = await this.db.createTable(TABLE_NAME, [seed]);
            // Create FTS index on text column
            await this.table.createIndex('text', {
                config: lancedb.Index.fts(),
            });
            // Delete the seed record
            await this.table.delete(`id = '__seed__'`);
        }
        this.initialized = true;
    }
    /**
     * Index or re-index a single entity.
     * Safe to call on create, update, or as part of bulk indexing.
     */
    async upsert(repoName, entity) {
        await this.init();
        const id = `${repoName}/${entity.entityType}/${entity.id}`;
        const text = buildEntityText(entity, this.readPdfSidecar(entity) ?? undefined);
        const vector = this.voyageApiKey
            ? await this.generateEmbedding(text)
            : null;
        const record = {
            id,
            repo: repoName,
            entity_type: entity.entityType,
            entity_id: entity.id,
            text,
            // Only include vector when we actually have one — avoids LanceDB schema inference issues
            vector: vector ?? new Array(VECTOR_DIM).fill(0),
            updated_at: new Date().toISOString(),
        };
        await this.table
            .mergeInsert('id')
            .whenMatchedUpdateAll()
            .whenNotMatchedInsertAll()
            .execute([record]);
    }
    /**
     * Bulk upsert — used by `studiograph index`. Batches Voyage API calls.
     */
    async upsertMany(repoName, entities, opts = {}) {
        await this.init();
        if (entities.length === 0)
            return;
        const texts = entities.map(e => buildEntityText(e, this.readPdfSidecar(e) ?? undefined));
        // Batch embeddings (Voyage supports up to 128 per request)
        let vectors = new Array(entities.length).fill(null);
        if (this.voyageApiKey) {
            vectors = await this.generateEmbeddingsBatch(texts, opts.onProgress);
        }
        else {
            // No embedding step — report the whole batch as "done" once so
            // progress UIs that wait on at least one callback don't sit at 0%.
            opts.onProgress?.(entities.length, entities.length);
        }
        const records = entities.map((entity, i) => ({
            id: `${repoName}/${entity.entityType}/${entity.id}`,
            repo: repoName,
            entity_type: entity.entityType,
            entity_id: entity.id,
            text: texts[i],
            vector: vectors[i] ?? new Array(VECTOR_DIM).fill(0),
            updated_at: new Date().toISOString(),
        }));
        await this.table
            .mergeInsert('id')
            .whenMatchedUpdateAll()
            .whenNotMatchedInsertAll()
            .execute(records);
    }
    /**
     * Remove an entity from the index.
     */
    async delete(repoName, entityType, entityId) {
        await this.init();
        const id = `${repoName}/${entityType}/${entityId}`;
        await this.table.delete(`id = '${id.replace(/'/g, "''")}'`);
    }
    /**
     * Remove an entity by `(repo, id)` alone, without needing its type.
     *
     * Used by the serve-boot incremental indexer: when git diff reports a
     * deletion, the file is gone from disk so its type can't be resolved
     * from frontmatter. The vector store has a discrete `entity_id` column
     * so we can match on `repo + entity_id` without falling back to a LIKE
     * pattern on the synthetic id.
     *
     * Safe across type renames: a single id that legitimately had two type
     * rows (e.g. type changed mid-edit and both old + new still exist) gets
     * both wiped — which is the right behavior for incremental rebuild
     * since the next upsert recreates the current type's row.
     */
    async deleteByRepoAndId(repoName, entityId) {
        await this.init();
        const r = repoName.replace(/'/g, "''");
        const e = entityId.replace(/'/g, "''");
        await this.table.delete(`repo = '${r}' AND entity_id = '${e}'`);
    }
    /**
     * Search the index.
     * Uses hybrid search (vector + FTS) when Voyage key is configured, FTS only otherwise.
     */
    async search(query, options = {}) {
        await this.init();
        const limit = options.limit ?? 10;
        try {
            let rows;
            if (this.voyageApiKey) {
                // Tier 2: hybrid search
                const queryVector = await this.generateEmbedding(query);
                rows = await this.table
                    .search(queryVector)
                    .limit(limit * 2) // fetch extra for re-ranking
                    .toArray();
            }
            else {
                // Tier 1: FTS only
                rows = await this.table
                    .search(query, 'text')
                    .limit(limit)
                    .toArray();
            }
            let results = rows
                .filter((r) => r.id !== '__seed__' && r.entity_id !== '')
                .map((r, i) => ({
                repoName: r.repo,
                entityType: r.entity_type,
                entityId: r.entity_id,
                score: r._distance != null ? 1 / (1 + r._distance) : 1 - i * 0.01,
            }));
            if (options.repos && options.repos.length > 0) {
                results = results.filter(r => options.repos.includes(r.repoName));
            }
            return results.slice(0, limit);
        }
        catch {
            // FTS index may not be built yet — return empty
            return [];
        }
    }
    /**
     * Get the last time a repo's entities were indexed.
     */
    getLastIndexedTime(repoName) {
        if (!existsSync(this.lastIndexedPath))
            return undefined;
        try {
            const data = JSON.parse(readFileSync(this.lastIndexedPath, 'utf-8'));
            return data[repoName];
        }
        catch {
            return undefined;
        }
    }
    /**
     * Record the last index time for a repo.
     *
     * Production callers omit `isoTimestamp` and the current time is used.
     * Tests pass an explicit ISO timestamp so they can rewind the clock and
     * assert the git-log `--since` window picks up the right commits.
     */
    setLastIndexedTime(repoName, isoTimestamp) {
        mkdirSync(this.dbPath, { recursive: true });
        let data = {};
        if (existsSync(this.lastIndexedPath)) {
            try {
                data = JSON.parse(readFileSync(this.lastIndexedPath, 'utf-8'));
            }
            catch { /* ignore */ }
        }
        data[repoName] = isoTimestamp ?? new Date().toISOString();
        writeFileSync(this.lastIndexedPath, JSON.stringify(data, null, 2) + '\n');
    }
    /** Whether Voyage AI embeddings are enabled (Tier 2). */
    get hasEmbeddings() {
        return !!this.voyageApiKey;
    }
    // ─── Private: Voyage AI ──────────────────────────────────────────────────────
    async generateEmbedding(text) {
        const results = await this.generateEmbeddingsBatch([text]);
        return results[0] ?? new Array(VECTOR_DIM).fill(0);
    }
    async generateEmbeddingsBatch(texts, onProgress) {
        // Voyage supports up to 128 inputs per request
        const BATCH_SIZE = 128;
        const all = [];
        const total = texts.length;
        for (let i = 0; i < texts.length; i += BATCH_SIZE) {
            const batch = texts.slice(i, i + BATCH_SIZE);
            const response = await fetch(VOYAGE_API_URL, {
                method: 'POST',
                headers: {
                    'Authorization': `Bearer ${this.voyageApiKey}`,
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({ model: VOYAGE_MODEL, input: batch }),
            });
            if (!response.ok) {
                const body = await response.text();
                throw new Error(`Voyage AI error ${response.status}: ${body}`);
            }
            const json = await response.json();
            all.push(...json.data.map(d => d.embedding));
            onProgress?.(all.length, total);
        }
        return all;
    }
}
// ─── Standalone reindex ──────────────────────────────────────────────────────
/**
 * Index (or re-index) all entities across workspace repos into LanceDB.
 * Standalone function — does not require a WorkspaceManager instance.
 * Used by `join`, `commit`, and other CLI commands that land entities on disk.
 *
 * @returns Total number of entities indexed.
 */
export async function reindexWorkspace(workspacePath, repos, onProgress) {
    const { loadUserConfig } = await import('../core/user-config.js');
    const { BaseGraphManager } = await import('../core/graph.js');
    const { homedir } = await import('os');
    const userConfig = loadUserConfig();
    const dbPath = userConfig.vector_db_path || join(homedir(), '.studiograph', 'vector');
    const vectorService = new VectorService(dbPath, userConfig.voyage_api_key, workspacePath);
    await vectorService.init();
    const log = onProgress ?? (() => { });
    let total = 0;
    for (const repo of repos) {
        const repoPath = join(workspacePath, repo.path);
        if (!existsSync(repoPath))
            continue;
        try {
            const graph = new BaseGraphManager({
                repoPath,
                repoName: repo.name,
                gitUser: { id: 'reindex', name: 'reindex', email: 'reindex@studiograph.local' },
            });
            const entities = graph.list();
            if (entities.length > 0) {
                await vectorService.upsertMany(repo.name, entities);
                total += entities.length;
            }
            vectorService.setLastIndexedTime(repo.name);
            log(`  Indexed ${repo.name}: ${entities.length} entities`);
        }
        catch (err) {
            log(`  Failed to index ${repo.name}: ${err instanceof Error ? err.message : 'unknown'}`);
        }
    }
    return total;
}
export async function incrementalReindexWorkspace(workspaceManager, repos, workspacePath, opts = {}) {
    const vectorService = workspaceManager.vectorService;
    if (!vectorService) {
        return { total: 0, fresh: false };
    }
    const log = opts.log ?? (() => { });
    const { execFileSync } = await import('child_process');
    let total = 0;
    let anyFreshRepo = false;
    const staleEntities = [];
    // Pre-pass: figure out the total first-time entity count so the
    // progress callback has a stable denominator. Cheap — `graph.list()`
    // doesn't re-read files, it returns the in-memory graph state the
    // workspace manager already maintains.
    let firstTimeTotal = 0;
    if (opts.onFirstTimeProgress) {
        for (const repo of repos) {
            if (vectorService.getLastIndexedTime(repo.name))
                continue;
            try {
                firstTimeTotal += workspaceManager.getGraph(repo.name).list().length;
            }
            catch {
                // tolerate per-repo failure; the real loop below will log it
            }
        }
    }
    let firstTimeDone = 0;
    for (const repo of repos) {
        const lastIndexed = vectorService.getLastIndexedTime(repo.name);
        // Per-repo first-time index. Catches three cases:
        //   1. Very first `serve` boot (every repo missing a timestamp).
        //   2. New folder added between serve boots (existing workspace +
        //      new repo with no timestamp). Pre-fix this was skipped here
        //      and never indexed unless the operator ran a full reindex.
        //   3. Vector DB rebuilt from scratch but workspace has existing
        //      folders.
        if (!lastIndexed) {
            try {
                const graph = workspaceManager.getGraph(repo.name);
                const entities = graph.list();
                if (entities.length > 0) {
                    await vectorService.upsertMany(repo.name, entities, {
                        onProgress: (batchDone) => {
                            opts.onFirstTimeProgress?.(firstTimeDone + batchDone, firstTimeTotal);
                        },
                    });
                    firstTimeDone += entities.length;
                    total += entities.length;
                }
                vectorService.setLastIndexedTime(repo.name);
                anyFreshRepo = true;
            }
            catch (err) {
                log(`failed to index ${repo.name}: ${err instanceof Error ? err.message : 'unknown'}`);
            }
            continue;
        }
        // Established repo — diff git log since the last index timestamp and
        // collect a list of stale (repo, id) tuples to upsert / delete below.
        const repoPath = join(workspacePath, repo.path);
        if (!existsSync(repoPath))
            continue;
        try {
            const output = execFileSync('git', ['log', `--since=${lastIndexed}`, '--name-status', '--format=', '--', '*.md'], { cwd: repoPath, encoding: 'utf-8', stdio: 'pipe' }).trim();
            if (!output)
                continue;
            for (const line of output.split('\n')) {
                // Accept any markdown path. Layouts in the wild:
                //   - flat:    `entity-id.md`
                //   - typed:   `<type>/entity-id.md`
                //   - nested:  `folder/sub/entity-id.md`
                // STU-131 normalized on flat for new entities; older typed layouts
                // and arbitrary nested folders still appear in mixed repos. Entity
                // type lives in frontmatter (resolved via graph.getById for
                // upserts; not needed for deletes since we now match by `(repo,
                // entity_id)` columns). Pre-fix this regex required exactly one
                // slash with a `\w+` type segment and missed all three layouts,
                // plus hyphenated types like `case-study`.
                const m = line.match(/^([AMD])\s+(.+\.md)$/);
                if (!m)
                    continue;
                const [, status, path] = m;
                // basename minus `.md`. Entity IDs are unique within a repo.
                const id = path.split('/').pop().replace(/\.md$/, '');
                if (!id)
                    continue;
                staleEntities.push({ repoName: repo.name, entityId: id, deleted: status === 'D' });
            }
        }
        catch (err) {
            log(`failed to diff ${repo.name}: ${err instanceof Error ? err.message : 'unknown'}`);
        }
    }
    if (staleEntities.length === 0) {
        // Nothing changed via git diff. If we did first-time work above, that
        // counts as a fresh run; otherwise this was a no-op incremental boot.
        return { total, fresh: anyFreshRepo };
    }
    // Dedupe: if an entity appears multiple times (e.g. M then D), the last
    // entry wins. Order matters — git log is newest-first by default, so we
    // reverse before reducing to get the newest state per (repo, id).
    const seen = new Map();
    for (const item of staleEntities.slice().reverse()) {
        seen.set(`${item.repoName}/${item.entityId}`, item);
    }
    for (const item of seen.values()) {
        try {
            if (item.deleted) {
                // Delete by (repo, id) — type isn't recoverable from a deleted file.
                // The new column-based matcher in vectorService handles it without
                // a frontmatter lookup.
                await vectorService.deleteByRepoAndId(item.repoName, item.entityId);
            }
            else {
                const graph = workspaceManager.getGraph(item.repoName);
                const entity = graph.getById(item.entityId);
                await vectorService.upsert(item.repoName, entity);
                total++;
            }
        }
        catch {
            // Entity may have been deleted then re-created; safe to skip.
        }
    }
    // Stamp every repo we touched. Empty-diff repos already had a timestamp
    // (we skipped the rebuild path) — re-stamping is cheap and keeps the
    // "since" window tight for the next boot.
    for (const repo of repos) {
        vectorService.setLastIndexedTime(repo.name);
    }
    return { total, fresh: anyFreshRepo };
}
// ─── Helpers ─────────────────────────────────────────────────────────────────
/**
 * Build a text representation of an entity for FTS/embedding.
 * Combines entity type, key frontmatter fields, and markdown body.
 *
 * `extraText` is appended after the body — used by VectorService to
 * fold PDF sidecar text into asset-entity embeddings. Kept as a plain
 * string parameter (not file IO) so this stays pure and unit-testable.
 *
 * Exported for testing.
 */
export function buildEntityText(entity, extraText) {
    const parts = [];
    // Entity type and ID as context
    parts.push(`${entity.entityType}: ${entity.data.name || entity.id}`);
    // Key frontmatter fields (skip system fields and empty values)
    const systemFields = new Set([
        'entity_type', 'entity_id', 'created_at', 'updated_at',
        'created_by', 'updated_by', 'visibility', 'restricted_to',
        'sensitive_fields',
    ]);
    for (const [key, value] of Object.entries(entity.data)) {
        if (systemFields.has(key) || value == null || value === '')
            continue;
        if (Array.isArray(value)) {
            if (value.length > 0)
                parts.push(`${key}: ${value.join(', ')}`);
        }
        else {
            parts.push(`${key}: ${String(value)}`);
        }
    }
    // Markdown body
    if (entity.document.content) {
        // Strip markdown syntax for cleaner FTS/embedding input
        const plainText = entity.document.content
            .replace(/^#{1,6}\s+/gm, '') // headings
            .replace(/\*\*(.+?)\*\*/g, '$1') // bold
            .replace(/\*(.+?)\*/g, '$1') // italic
            .replace(/`(.+?)`/g, '$1') // inline code
            .replace(/\[\[(.+?)\]\]/g, '$1') // wikilinks
            .trim();
        if (plainText)
            parts.push(plainText);
    }
    if (extraText) {
        const trimmed = extraText.trim();
        if (trimmed)
            parts.push(trimmed);
    }
    return parts.join('\n');
}
//# sourceMappingURL=vector-service.js.map