/**
 * Intelligence MCP Tools
 *
 * Tools: ocr_guide, ocr_document_tables, ocr_document_recommend, ocr_document_extras
 *
 * Internal-only data access and analysis tools. No external API calls needed.
 *
 * CRITICAL: NEVER use console.log() - stdout is reserved for JSON-RPC protocol.
 * Use console.error() for all logging.
 *
 * @module tools/intelligence
 */
import { z } from 'zod';
import { state, hasDatabase, requireDatabase, getDefaultStoragePath } from '../server/state.js';
import { DatabaseService } from '../services/storage/database/index.js';
import { successResult } from '../server/types.js';
import { validateInput } from '../utils/validation.js';
import { MCPError, documentNotFoundError } from '../server/errors.js';
import { formatResponse, handleError } from './shared.js';
import { RegistryService } from '../services/storage/registry/index.js';
// ═══════════════════════════════════════════════════════════════════════════════
// INPUT SCHEMAS
// ═══════════════════════════════════════════════════════════════════════════════
const GuideInput = z.object({
    intent: z
        .enum(['explore', 'search', 'ingest', 'analyze', 'status'])
        .optional()
        .describe('Optional intent hint: explore (browse data), search (find content), ingest (add documents), analyze (compare/cluster), status (check health). Omit for general guidance.'),
});
const DocumentTablesInput = z.object({
    document_id: z.string().min(1).describe('Document ID to extract tables from'),
    table_index: z
        .number()
        .int()
        .min(0)
        .optional()
        .describe('Specific table index (0-based) to retrieve. Omit for all tables.'),
    limit: z
        .number()
        .int()
        .min(1)
        .max(50)
        .default(10)
        .describe('Maximum tables to return (default 10)'),
    offset: z
        .number()
        .int()
        .min(0)
        .default(0)
        .describe('Number of tables to skip for pagination'),
    include_cells: z
        .boolean()
        .default(false)
        .describe('Include cell data. Default returns table summary only (caption, row_count, column_count). Use with table_index for a specific table.'),
});
const DocumentRecommendInput = z.object({
    document_id: z.string().min(1).describe('Source document ID to get recommendations for'),
    limit: z.number().int().min(1).max(50).default(10).describe('Maximum number of recommendations'),
});
const DocumentExtrasInput = z.object({
    document_id: z.string().min(1).describe('Document ID to retrieve extras data for'),
    section: z
        .string()
        .optional()
        .describe('Specific extras section to retrieve (charts, links, tracked_changes, table_row_bboxes, infographics). Omit for section manifest with counts.'),
    limit: z
        .number()
        .int()
        .min(1)
        .max(200)
        .default(50)
        .describe('Maximum items when section data is an array (default 50)'),
    offset: z
        .number()
        .int()
        .min(0)
        .default(0)
        .describe('Number of items to skip when section data is an array'),
});
// ═══════════════════════════════════════════════════════════════════════════════
// TABLE EXTRACTION HELPERS
// ═══════════════════════════════════════════════════════════════════════════════
/**
 * Walk JSON blocks tree looking for Table-type blocks.
 * Extracts cell data into a structured format.
 */
function extractTablesFromBlocks(blocks) {
    const tables = [];
    function walkBlock(block, pageNumber) {
        const blockType = block.block_type;
        // Track page number from Page blocks (handle both number and numeric string IDs)
        let currentPage = pageNumber;
        if (blockType === 'Page') {
            if (typeof block.id === 'number') {
                currentPage = block.id + 1;
            }
            else if (typeof block.id === 'string' && /^\d+$/.test(block.id)) {
                currentPage = parseInt(block.id, 10) + 1;
            }
            else if (typeof block.page === 'number') {
                currentPage = block.page + 1;
            }
        }
        // Fallback: if block has a page field, use it
        if (currentPage === null && typeof block.page === 'number') {
            currentPage = block.page + 1;
        }
        if (blockType === 'Table') {
            const table = parseTableBlock(block, tables.length, currentPage);
            tables.push(table);
        }
        // Recurse into children
        const children = block.children;
        if (Array.isArray(children)) {
            for (const child of children) {
                walkBlock(child, currentPage);
            }
        }
    }
    for (const block of blocks) {
        walkBlock(block, null);
    }
    return tables;
}
/**
 * Parse a single Table block into a structured table representation.
 */
function parseTableBlock(block, tableIndex, pageNumber) {
    const cells = [];
    let maxRow = 0;
    let maxCol = 0;
    let caption = null;
    // Look for caption in the block itself or nearby
    if (typeof block.html === 'string' && block.html.includes('<caption>')) {
        const captionMatch = block.html.match(/<caption>(.*?)<\/caption>/);
        if (captionMatch) {
            caption = captionMatch[1];
        }
    }
    // Try to extract cells from HTML if available
    if (typeof block.html === 'string') {
        const html = block.html;
        extractCellsFromHTML(html, cells);
    }
    // Also try to extract from children blocks (TableRow/TableCell pattern)
    const children = block.children;
    if (Array.isArray(children) && cells.length === 0) {
        let rowIndex = 0;
        for (const child of children) {
            const childType = child.block_type;
            if (childType === 'TableRow' || childType === 'TableHeader') {
                const rowChildren = child.children;
                if (Array.isArray(rowChildren)) {
                    let colIndex = 0;
                    for (const cell of rowChildren) {
                        const cellType = cell.block_type;
                        if (cellType === 'TableCell' || cellType === 'TableHeaderCell') {
                            const text = extractBlockText(cell);
                            cells.push({ row: rowIndex, col: colIndex, text });
                            if (colIndex > maxCol)
                                maxCol = colIndex;
                            colIndex++;
                        }
                    }
                }
                rowIndex++;
            }
        }
        maxRow = rowIndex > 0 ? rowIndex - 1 : 0;
    }
    // Compute maxRow/maxCol from cells
    for (const cell of cells) {
        if (cell.row > maxRow)
            maxRow = cell.row;
        if (cell.col > maxCol)
            maxCol = cell.col;
    }
    return {
        table_index: tableIndex,
        page_number: pageNumber,
        caption,
        row_count: cells.length > 0 ? maxRow + 1 : 0,
        column_count: cells.length > 0 ? maxCol + 1 : 0,
        cells,
    };
}
/**
 * Extract cells from HTML table string.
 */
function extractCellsFromHTML(html, cells) {
    // Split by rows
    const rowRegex = /<tr[^>]*>([\s\S]*?)<\/tr>/gi;
    let rowMatch;
    let rowIndex = 0;
    while ((rowMatch = rowRegex.exec(html)) !== null) {
        const rowContent = rowMatch[1];
        const cellRegex = /<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi;
        let cellMatch;
        let colIndex = 0;
        while ((cellMatch = cellRegex.exec(rowContent)) !== null) {
            // Strip inner HTML tags to get text
            const text = cellMatch[1].replace(/<[^>]*>/g, '').trim();
            cells.push({ row: rowIndex, col: colIndex, text });
            colIndex++;
        }
        rowIndex++;
    }
}
/**
 * Extract text content from a block recursively.
 */
function extractBlockText(block) {
    if (typeof block.text === 'string')
        return block.text;
    if (typeof block.html === 'string') {
        return block.html.replace(/<[^>]*>/g, '').trim();
    }
    const children = block.children;
    if (Array.isArray(children)) {
        return children.map(extractBlockText).filter(Boolean).join(' ');
    }
    return '';
}
/**
 * Fetch and parse json_blocks for a document from ocr_results.
 * Handles both formats: flat array or {children: [...], metadata: {...}}.
 */
function fetchJsonBlocks(conn, documentId) {
    const ocrRow = conn
        .prepare('SELECT json_blocks FROM ocr_results WHERE document_id = ?')
        .get(documentId);
    if (!ocrRow?.json_blocks) {
        console.error(`[intelligence] json_blocks is null for document ${documentId}. ` +
            `This is expected for DOCX files where OCR processing does not return JSON block data. ` +
            `Falling back to chunk-based table extraction.`);
        return { ok: false, reason: 'no_ocr_data' };
    }
    let blocks;
    try {
        const parsed = JSON.parse(ocrRow.json_blocks);
        if (Array.isArray(parsed)) {
            blocks = parsed;
        }
        else if (parsed &&
            typeof parsed === 'object' &&
            Array.isArray(parsed.children)) {
            blocks = parsed.children;
        }
        else {
            blocks = [];
        }
    }
    catch (parseErr) {
        console.error(`[intelligence] Failed to parse json_blocks for ${documentId}: ${String(parseErr)}`);
        return { ok: false, reason: 'parse_error' };
    }
    if (blocks.length === 0) {
        return { ok: false, reason: 'empty' };
    }
    return { ok: true, blocks };
}
/**
 * Filter parsed tables by optional table_index.
 * Returns null if the index is out of range (caller should handle).
 */
function filterTablesByIndex(allTables, tableIndex) {
    if (tableIndex === undefined) {
        return allTables;
    }
    if (tableIndex >= allTables.length) {
        return null;
    }
    return [allTables[tableIndex]];
}
/**
 * Parse tables from chunk metadata when json_blocks is unavailable (e.g., DOCX files).
 *
 * Queries chunks tagged with table content types or having table_columns in
 * their provenance processing_params, then parses pipe-delimited markdown
 * text into structured ParsedTable objects.
 *
 * @param conn - Database connection
 * @param documentId - Document ID to extract tables from
 * @param callerLabel - Label for error messages (e.g., 'DocumentTables', 'TableExport')
 * @returns Array of ParsedTable objects extracted from chunk text
 */
function parseTablesFromChunks(conn, documentId, callerLabel) {
    const tableChunks = conn
        .prepare(`SELECT c.id, c.text, c.page_number, c.chunk_index,
              p.processing_params
       FROM chunks c
       LEFT JOIN provenance p ON c.provenance_id = p.id
       WHERE c.document_id = ? AND (
         c.content_types LIKE '%table%'
         OR p.processing_params LIKE '%table_columns%'
       )
       ORDER BY c.chunk_index ASC`)
        .all(documentId);
    return tableChunks.map((tc, idx) => {
        let tableSummary = null;
        let columnCount = 0;
        let rowCount = 0;
        if (tc.processing_params) {
            try {
                const pp = JSON.parse(tc.processing_params);
                tableSummary = pp.table_summary ?? null;
                columnCount = pp.table_column_count ?? 0;
                rowCount = pp.table_row_count ?? 0;
            }
            catch (parseErr) {
                console.error(`[${callerLabel}] Failed to parse processing_params for chunk ${tc.id}: ${String(parseErr)}`);
            }
        }
        // Parse markdown table text to extract cells
        const cells = [];
        const lines = tc.text.split('\n').filter((l) => l.trim().length > 0);
        let rowIdx = 0;
        for (const line of lines) {
            // Skip separator lines (e.g., |---|---|)
            if (/^\|[\s\-:|]+\|$/.test(line.trim()))
                continue;
            if (!line.includes('|'))
                continue;
            const rawCells = line
                .split('|')
                .map((c) => c.trim())
                .filter((c) => c.length > 0);
            for (let colIdx = 0; colIdx < rawCells.length; colIdx++) {
                cells.push({ row: rowIdx, col: colIdx, text: rawCells[colIdx] });
            }
            if (rawCells.length > 0) {
                if (rawCells.length > columnCount)
                    columnCount = rawCells.length;
                rowIdx++;
            }
        }
        if (rowIdx > rowCount)
            rowCount = rowIdx;
        return {
            table_index: idx,
            page_number: tc.page_number,
            caption: tableSummary,
            row_count: rowCount,
            column_count: columnCount,
            cells,
        };
    });
}
// ═══════════════════════════════════════════════════════════════════════════════
// HANDLER: ocr_guide
// ═══════════════════════════════════════════════════════════════════════════════
/**
 * Handle ocr_guide - Contextual navigation aid for AI agents.
 *
 * Inspects current system state (databases, selected DB, document counts,
 * processing status) and returns actionable guidance. No external API calls.
 */
async function handleGuide(params) {
    try {
        const input = validateInput(GuideInput, params);
        const intent = input.intent;
        const storagePath = getDefaultStoragePath();
        const databases = DatabaseService.list(storagePath).filter(db => !db.name.startsWith('_'));
        const selectedDb = state.currentDatabaseName;
        const dbSelected = hasDatabase();
        // Build context about current state
        const context = {
            databases_available: databases.length,
            database_names: databases.map((d) => d.name),
            selected_database: selectedDb ?? 'none',
        };
        // Add recently used databases from registry
        let recentlyUsed = [];
        try {
            const registry = RegistryService.getInstance();
            const recent = registry.getRecent(5);
            recentlyUsed = recent.map(r => ({
                name: r.name,
                last_accessed: r.last_accessed_at,
                documents: r.document_count,
                description: r.description,
            }));
        }
        catch (registryError) {
            console.error(`[guide] Registry query failed: ${registryError instanceof Error ? registryError.message : String(registryError)}`);
        }
        context.recently_used = recentlyUsed;
        // If a database is selected, get its stats
        let docCount = 0;
        let pendingCount = 0;
        let completeCount = 0;
        let failedCount = 0;
        let chunkCount = 0;
        let embeddingCount = 0;
        let imageCount = 0;
        let clusterCount = 0;
        let embeddingCoverage = 0;
        let vlmCoverage = 0;
        if (dbSelected) {
            try {
                const { db } = requireDatabase();
                const conn = db.getConnection();
                const statusRows = conn
                    .prepare('SELECT status, COUNT(*) as count FROM documents GROUP BY status')
                    .all();
                for (const row of statusRows) {
                    docCount += row.count;
                    if (row.status === 'pending')
                        pendingCount = row.count;
                    else if (row.status === 'complete')
                        completeCount = row.count;
                    else if (row.status === 'failed')
                        failedCount = row.count;
                }
                chunkCount = conn.prepare('SELECT COUNT(*) as c FROM chunks').get().c;
                embeddingCount = conn.prepare('SELECT COUNT(*) as c FROM embeddings').get().c;
                imageCount = conn.prepare('SELECT COUNT(*) as c FROM images').get().c;
                clusterCount = conn.prepare('SELECT COUNT(*) as c FROM clusters').get()
                    .c;
                context.database_stats = {
                    total_documents: docCount,
                    complete: completeCount,
                    pending: pendingCount,
                    failed: failedCount,
                    chunks: chunkCount,
                    embeddings: embeddingCount,
                    images: imageCount,
                    clusters: clusterCount,
                    image_embeddings: conn.prepare('SELECT COUNT(*) as c FROM embeddings WHERE image_id IS NOT NULL').get().c,
                };
                // V7: Corpus snapshot for smarter guide
                if (docCount > 0) {
                    const fileTypeRows = conn
                        .prepare('SELECT file_type, COUNT(*) as count FROM documents WHERE file_type IS NOT NULL GROUP BY file_type ORDER BY count DESC')
                        .all();
                    const comparisonCount = conn.prepare('SELECT COUNT(*) as c FROM comparisons').get().c;
                    embeddingCoverage = chunkCount > 0 ? Math.round((embeddingCount / chunkCount) * 100) : 0;
                    // Count images with VLM descriptions vs total
                    const vlmCompleteCount = imageCount > 0
                        ? conn
                            .prepare("SELECT COUNT(*) as c FROM images WHERE vlm_status = 'complete'")
                            .get().c
                        : 0;
                    vlmCoverage = imageCount > 0 ? Math.round((vlmCompleteCount / imageCount) * 100) : 0;
                    context.corpus_snapshot = {
                        document_count: docCount,
                        total_chunks: chunkCount,
                        total_images: imageCount,
                        file_types: fileTypeRows.map((r) => r.file_type),
                        has_clusters: clusterCount > 0,
                        has_comparisons: comparisonCount > 0,
                        embedding_coverage: `${embeddingCoverage}%`,
                        vlm_coverage: `${vlmCoverage}%`,
                    };
                }
            }
            catch (err) {
                const errMsg = err instanceof Error ? err.message : String(err);
                throw new Error(`Database "${selectedDb}" selected but query failed: ${errMsg}`);
            }
        }
        // Build next_steps based on state and intent
        const next_steps = [];
        if (!dbSelected) {
            if (databases.length === 0) {
                next_steps.push({
                    tool: 'ocr_db_create',
                    description: 'Create a database first, then ingest documents.',
                    priority: 'required',
                });
            }
            else {
                if (recentlyUsed.length > 0) {
                    next_steps.unshift({
                        tool: 'ocr_db_select',
                        description: `Resume work on "${recentlyUsed[0].name}" (last used ${recentlyUsed[0].last_accessed})`,
                        priority: 'suggested',
                    });
                }
                next_steps.push({
                    tool: 'ocr_db_select',
                    description: 'Select a database to work with (see database_names in context above)',
                    priority: 'required',
                });
                next_steps.push({ tool: 'ocr_db_search', description: 'Search databases by name/description/tags', priority: 'available' });
            }
            return formatResponse(successResult({
                status: 'no_database_selected',
                message: databases.length === 0
                    ? 'No databases exist. Create one with ocr_db_create, then ingest documents.'
                    : `${databases.length} database(s) available. Select one with ocr_db_select to get started.`,
                context,
                next_steps,
            }));
        }
        // Database is selected - provide guidance based on intent and state
        if (intent === 'ingest' || (docCount === 0 && !intent)) {
            next_steps.push({
                tool: 'ocr_ingest_files',
                description: 'Ingest specific files by path.',
                priority: docCount === 0 ? 'required' : 'optional',
            });
            next_steps.push({
                tool: 'ocr_ingest_directory',
                description: 'Scan a directory for documents to ingest.',
                priority: 'optional',
            });
            if (pendingCount > 0) {
                next_steps.push({
                    tool: 'ocr_process_pending',
                    description: `Process ${pendingCount} pending documents through OCR pipeline.`,
                    priority: 'required',
                });
            }
        }
        else if (intent === 'search' || (!intent && completeCount > 0)) {
            next_steps.push({
                tool: 'ocr_search',
                description: 'Search across all documents. Use natural language questions as queries (e.g. "What does the author say about X?"), not keyword lists.',
                priority: 'recommended',
            });
            next_steps.push({
                tool: 'ocr_rag_context',
                description: 'Get pre-assembled context for answering a specific question. Pass the question in natural language.',
                priority: 'recommended',
            });
            if (embeddingCount === 0 && chunkCount > 0) {
                next_steps.push({
                    tool: 'ocr_health_check',
                    description: 'Chunks exist but no embeddings. Run health check with fix=true.',
                    priority: 'required',
                });
            }
        }
        else if (intent === 'explore') {
            next_steps.push({
                tool: 'ocr_document_list',
                description: `Browse ${docCount} documents in the database.`,
                priority: 'recommended',
            });
            next_steps.push({
                tool: 'ocr_report_overview',
                description: 'Get corpus overview with content type distribution (section="corpus").',
                priority: 'optional',
            });
        }
        else if (intent === 'analyze') {
            if (clusterCount > 0) {
                next_steps.push({
                    tool: 'ocr_cluster_list',
                    description: `View ${clusterCount} existing clusters.`,
                    priority: 'recommended',
                });
            }
            else if (completeCount >= 2) {
                next_steps.push({
                    tool: 'ocr_cluster_documents',
                    description: `Cluster ${completeCount} documents by similarity.`,
                    priority: 'recommended',
                });
            }
            if (completeCount >= 2) {
                next_steps.push({
                    tool: 'ocr_document_compare',
                    description: 'Compare two documents to find differences.',
                    priority: 'optional',
                });
            }
            next_steps.push({
                tool: 'ocr_document_duplicates',
                description: 'Find duplicate documents by hash or similarity.',
                priority: 'optional',
            });
        }
        else if (intent === 'status') {
            next_steps.push({
                tool: 'ocr_health_check',
                description: 'Check for data integrity issues.',
                priority: 'recommended',
            });
            next_steps.push({
                tool: 'ocr_db_stats',
                description: 'Get comprehensive database statistics.',
                priority: 'optional',
            });
            if (failedCount > 0) {
                next_steps.push({
                    tool: 'ocr_retry_failed',
                    description: `${failedCount} failed documents. Reset for reprocessing.`,
                    priority: 'recommended',
                });
            }
        }
        else {
            // General guidance when DB has data and no specific intent
            if (pendingCount > 0) {
                next_steps.push({
                    tool: 'ocr_process_pending',
                    description: `${pendingCount} documents awaiting processing.`,
                    priority: 'recommended',
                });
            }
            if (failedCount > 0) {
                next_steps.push({
                    tool: 'ocr_retry_failed',
                    description: `${failedCount} failed documents need attention.`,
                    priority: 'recommended',
                });
            }
            if (completeCount > 0) {
                next_steps.push({
                    tool: 'ocr_search',
                    description: 'Search across all documents.',
                    priority: 'recommended',
                });
            }
            next_steps.push({
                tool: 'ocr_document_list',
                description: `Browse ${docCount} documents.`,
                priority: 'optional',
            });
            // V7: Context-aware next_steps from corpus snapshot
            if (embeddingCoverage < 100 && chunkCount > 0) {
                next_steps.push({
                    tool: 'ocr_health_check',
                    description: `Check for processing gaps (${embeddingCoverage}% embedding coverage).`,
                    priority: 'recommended',
                });
            }
            if (clusterCount > 0) {
                next_steps.push({
                    tool: 'ocr_cluster_list',
                    description: `Explore ${clusterCount} topic clusters.`,
                    priority: 'optional',
                });
            }
        }
        // Build summary message
        const parts = [];
        parts.push(`Database "${selectedDb}" selected.`);
        parts.push(`${docCount} documents (${completeCount} complete, ${pendingCount} pending, ${failedCount} failed).`);
        if (chunkCount > 0)
            parts.push(`${chunkCount} chunks, ${embeddingCount} embeddings.`);
        if (imageCount > 0)
            parts.push(`${imageCount} images.`);
        if (clusterCount > 0)
            parts.push(`${clusterCount} clusters.`);
        return formatResponse(successResult({
            status: 'ready',
            message: parts.join(' '),
            context,
            next_steps,
            workflow_chains: docCount > 0
                ? [
                    {
                        name: 'find_and_read',
                        steps: ['ocr_search -> ocr_chunk_context -> ocr_document_page'],
                        description: 'Find content, expand context, read full page',
                    },
                    {
                        name: 'compare_documents',
                        steps: ['ocr_comparison_discover -> ocr_document_compare -> ocr_comparison_get'],
                        description: 'Find similar pairs, diff them, inspect results',
                    },
                    {
                        name: 'process_new',
                        steps: ['ocr_ingest_files -> ocr_process_pending -> ocr_health_check'],
                        description: 'Add files, run OCR pipeline, verify completeness',
                    },
                ]
                : undefined,
        }));
    }
    catch (error) {
        return handleError(error);
    }
}
// ═══════════════════════════════════════════════════════════════════════════════
// HANDLER: ocr_document_tables
// ═══════════════════════════════════════════════════════════════════════════════
/**
 * Handle ocr_document_tables - Extract table data from JSON blocks
 */
async function handleDocumentTables(params) {
    try {
        const input = validateInput(DocumentTablesInput, params);
        const { db } = requireDatabase();
        const doc = db.getDocument(input.document_id);
        if (!doc) {
            throw documentNotFoundError(input.document_id);
        }
        const baseNextSteps = [
            { tool: 'ocr_document_tables', description: 'Get cell data: include_cells=true, table_index=N' },
            { tool: 'ocr_table_export', description: 'Export table as CSV/JSON for analysis' },
            { tool: 'ocr_document_page', description: 'Read the page containing this table for full context' },
        ];
        /**
         * Apply pagination and include_cells stripping to a table array,
         * then return the formatted response.
         */
        const buildTableResponse = (allTables, source) => {
            // Apply table_index filter first
            const filtered = filterTablesByIndex(allTables, input.table_index);
            if (filtered === null) {
                return formatResponse(successResult({
                    document_id: input.document_id,
                    file_name: doc.file_name,
                    tables: [],
                    total_tables: allTables.length,
                    requested_index: input.table_index,
                    message: `Table index ${input.table_index} out of range. Document has ${allTables.length} table(s).`,
                    next_steps: baseNextSteps,
                }));
            }
            // Apply pagination
            const tblOffset = input.offset ?? 0;
            const tblLimit = input.limit ?? 10;
            const totalFiltered = filtered.length;
            const paginated = filtered.slice(tblOffset, tblOffset + tblLimit);
            const hasMore = tblOffset + tblLimit < totalFiltered;
            // Strip cells if not requested
            const outputTables = paginated.map((t) => {
                if (input.include_cells)
                    return t;
                // Return summary without cell data
                return {
                    table_index: t.table_index,
                    page_number: t.page_number,
                    caption: t.caption,
                    row_count: t.row_count,
                    column_count: t.column_count,
                    cell_count: t.cells.length,
                };
            });
            const nextSteps = [];
            if (hasMore) {
                nextSteps.push({
                    tool: 'ocr_document_tables',
                    description: `Get next page (offset=${tblOffset + tblLimit})`,
                });
            }
            nextSteps.push(...baseNextSteps);
            return formatResponse(successResult({
                document_id: input.document_id,
                file_name: doc.file_name,
                tables: outputTables,
                total_tables: allTables.length,
                returned: outputTables.length,
                offset: tblOffset,
                limit: tblLimit,
                has_more: hasMore,
                source,
                next_steps: nextSteps,
            }));
        };
        const blocksResult = fetchJsonBlocks(db.getConnection(), input.document_id);
        if (!blocksResult.ok) {
            // Fallback: extract table data from chunk metadata (authoritative source)
            const chunkTables = parseTablesFromChunks(db.getConnection(), input.document_id, 'DocumentTables');
            if (chunkTables.length === 0) {
                return formatResponse(successResult({
                    document_id: input.document_id,
                    file_name: doc.file_name,
                    tables: [],
                    total_tables: 0,
                    source: 'no_table_data',
                    next_steps: baseNextSteps,
                }));
            }
            return buildTableResponse(chunkTables, 'chunk_metadata');
        }
        const allTables = extractTablesFromBlocks(blocksResult.blocks);
        return buildTableResponse(allTables, 'json_blocks');
    }
    catch (error) {
        return handleError(error);
    }
}
// ═══════════════════════════════════════════════════════════════════════════════
// HANDLER: ocr_document_recommend
// ═══════════════════════════════════════════════════════════════════════════════
/**
 * Handle ocr_document_recommend - Cluster-based document recommendations
 *
 * Combines two signals:
 * 1. Cluster peers (documents in the same cluster)
 * 2. Vector similarity (centroid-based similar documents)
 */
async function handleDocumentRecommend(params) {
    try {
        const input = validateInput(DocumentRecommendInput, params);
        const { db, vector } = requireDatabase();
        // Verify source document exists
        const doc = db.getDocument(input.document_id);
        if (!doc) {
            throw documentNotFoundError(input.document_id);
        }
        const conn = db.getConnection();
        const limit = input.limit ?? 10;
        // Map of document_id -> recommendation entry
        const recommendations = new Map();
        // ──────────────────────────────────────────────────────────────
        // Signal 1: Cluster peers
        // ──────────────────────────────────────────────────────────────
        const sourceClusters = conn
            .prepare('SELECT cluster_id FROM document_clusters WHERE document_id = ?')
            .all(input.document_id);
        if (sourceClusters.length > 0) {
            const clusterIds = sourceClusters.map((c) => c.cluster_id);
            for (const clusterId of clusterIds) {
                const peers = conn
                    .prepare('SELECT document_id FROM document_clusters WHERE cluster_id = ? AND document_id != ?')
                    .all(clusterId, input.document_id);
                for (const peer of peers) {
                    const existing = recommendations.get(peer.document_id);
                    if (existing) {
                        existing.cluster_match = true;
                        existing.cluster_ids.push(clusterId);
                    }
                    else {
                        recommendations.set(peer.document_id, {
                            cluster_match: true,
                            cluster_ids: [clusterId],
                            similarity: null,
                        });
                    }
                }
            }
        }
        // ──────────────────────────────────────────────────────────────
        // Signal 2: Vector similarity (centroid approach)
        // ──────────────────────────────────────────────────────────────
        const embeddingRows = conn
            .prepare('SELECT id FROM embeddings WHERE document_id = ? AND chunk_id IS NOT NULL')
            .all(input.document_id);
        if (embeddingRows.length > 0) {
            const vectors = [];
            for (const row of embeddingRows) {
                const vec = vector.getVector(row.id);
                if (vec)
                    vectors.push(vec);
            }
            if (vectors.length > 0) {
                // Compute centroid
                const dims = 768;
                const centroid = new Float32Array(dims);
                for (const vec of vectors) {
                    for (let i = 0; i < dims; i++) {
                        centroid[i] += vec[i];
                    }
                }
                for (let i = 0; i < dims; i++) {
                    centroid[i] /= vectors.length;
                }
                // Search for similar embeddings
                const searchResults = vector.searchSimilar(centroid, {
                    limit: limit * 10,
                    threshold: 0.4,
                });
                // Aggregate by document
                const docSimilarityMap = new Map();
                for (const r of searchResults) {
                    if (r.document_id === input.document_id)
                        continue;
                    const entry = docSimilarityMap.get(r.document_id);
                    if (entry) {
                        entry.totalSim += r.similarity_score;
                        entry.count += 1;
                    }
                    else {
                        docSimilarityMap.set(r.document_id, { totalSim: r.similarity_score, count: 1 });
                    }
                }
                for (const [docId, { totalSim, count }] of docSimilarityMap.entries()) {
                    const avgSim = Math.round((totalSim / count) * 1000000) / 1000000;
                    const existing = recommendations.get(docId);
                    if (existing) {
                        existing.similarity = avgSim;
                    }
                    else {
                        recommendations.set(docId, {
                            cluster_match: false,
                            cluster_ids: [],
                            similarity: avgSim,
                        });
                    }
                }
            }
        }
        // ──────────────────────────────────────────────────────────────
        // Merge, score, and rank
        // ──────────────────────────────────────────────────────────────
        const ranked = [];
        for (const [docId, rec] of recommendations.entries()) {
            const recDoc = db.getDocument(docId);
            // Score: cluster match = 0.5 bonus, similarity = actual value
            const clusterBonus = rec.cluster_match ? 0.5 : 0;
            const simScore = rec.similarity ?? 0;
            const score = Math.round((clusterBonus + simScore) * 1000000) / 1000000;
            const reasons = [];
            if (rec.cluster_match) {
                reasons.push(`cluster_peer (clusters: ${rec.cluster_ids.join(', ')})`);
            }
            if (rec.similarity !== null) {
                reasons.push(`similar (score: ${rec.similarity})`);
            }
            ranked.push({
                document_id: docId,
                file_name: recDoc?.file_name ?? null,
                file_type: recDoc?.file_type ?? null,
                status: recDoc?.status ?? null,
                score,
                reasons,
                cluster_match: rec.cluster_match,
                similarity: rec.similarity,
            });
        }
        ranked.sort((a, b) => b.score - a.score);
        const topRanked = ranked.slice(0, limit);
        return formatResponse(successResult({
            source_document_id: input.document_id,
            source_file_name: doc.file_name,
            source_cluster_count: sourceClusters.length,
            source_embedding_count: embeddingRows.length,
            recommendations: topRanked,
            total_candidates: ranked.length,
            returned: topRanked.length,
            next_steps: [
                { tool: 'ocr_document_get', description: 'Get details for a recommended document' },
                {
                    tool: 'ocr_document_compare',
                    description: 'Compare the source document with a recommendation',
                },
            ],
        }));
    }
    catch (error) {
        return handleError(error);
    }
}
// ═══════════════════════════════════════════════════════════════════════════════
// HANDLER: ocr_document_extras
// ═══════════════════════════════════════════════════════════════════════════════
/** Known extras sections */
const KNOWN_EXTRAS_SECTIONS = [
    'charts',
    'links',
    'tracked_changes',
    'table_row_bboxes',
    'infographics',
    'extras_features',
    'metadata',
    'cost_breakdown',
    'block_type_stats',
    'structural_fingerprint',
    'structured_links',
    'link_count',
];
/**
 * Handle ocr_document_extras - Surface extras_json data from OCR results
 */
async function handleDocumentExtras(params) {
    try {
        const input = validateInput(DocumentExtrasInput, params);
        const { db } = requireDatabase();
        // Verify document exists
        const doc = db.getDocument(input.document_id);
        if (!doc) {
            throw documentNotFoundError(input.document_id);
        }
        // Get extras_json from ocr_results
        const ocrRow = db
            .getConnection()
            .prepare('SELECT extras_json FROM ocr_results WHERE document_id = ?')
            .get(input.document_id);
        if (!ocrRow?.extras_json) {
            return formatResponse(successResult({
                document_id: input.document_id,
                file_name: doc.file_name,
                extras: {},
                available_sections: [],
                message: 'No extras data available for this document.',
                next_steps: [{ tool: 'ocr_document_get', description: 'View document details' }],
            }));
        }
        let extras;
        try {
            extras = JSON.parse(ocrRow.extras_json);
        }
        catch (parseErr) {
            console.error(`[DocumentExtras] Failed to parse extras_json for ${input.document_id}: ${String(parseErr)}`);
            throw new MCPError('INTERNAL_ERROR', `Failed to parse extras_json: ${String(parseErr)}`);
        }
        // Flatten extras_features children to the top level so fields like
        // links, table_row_bboxes, tracked_changes, charts, infographics are
        // directly accessible as sections rather than buried under extras_features.
        const extrasFeatures = extras.extras_features;
        if (extrasFeatures && typeof extrasFeatures === 'object') {
            for (const [key, value] of Object.entries(extrasFeatures)) {
                // Only promote if not already present at top level (avoid overwriting
                // enriched data like structured_links which may have been derived from
                // the raw extras_features.links)
                if (!(key in extras)) {
                    extras[key] = value;
                }
            }
        }
        // Determine available sections
        const availableSections = Object.keys(extras).filter((key) => extras[key] !== null && extras[key] !== undefined);
        // Filter by specific section if requested
        if (input.section) {
            if (!KNOWN_EXTRAS_SECTIONS.includes(input.section) &&
                !(input.section in extras)) {
                throw new MCPError('VALIDATION_ERROR', `Unknown section "${input.section}". Available sections: ${availableSections.join(', ')}`);
            }
            const sectionData = extras[input.section];
            // Apply limit/offset when section data is an array
            if (Array.isArray(sectionData)) {
                const extOffset = input.offset ?? 0;
                const extLimit = input.limit ?? 50;
                const totalItems = sectionData.length;
                const paginated = sectionData.slice(extOffset, extOffset + extLimit);
                const hasMore = extOffset + extLimit < totalItems;
                const nextSteps = [];
                if (hasMore) {
                    nextSteps.push({
                        tool: 'ocr_document_extras',
                        description: `Get next page: section='${input.section}', offset=${extOffset + extLimit}`,
                    });
                }
                nextSteps.push({ tool: 'ocr_document_tables', description: 'Extract table data from the document' }, { tool: 'ocr_document_get', description: 'View core document metadata' });
                return formatResponse(successResult({
                    document_id: input.document_id,
                    file_name: doc.file_name,
                    section: input.section,
                    data: paginated,
                    total_items: totalItems,
                    returned: paginated.length,
                    offset: extOffset,
                    limit: extLimit,
                    has_more: hasMore,
                    available_sections: availableSections,
                    next_steps: nextSteps,
                }));
            }
            return formatResponse(successResult({
                document_id: input.document_id,
                file_name: doc.file_name,
                section: input.section,
                data: sectionData ?? null,
                available_sections: availableSections,
                next_steps: [
                    { tool: 'ocr_document_tables', description: 'Extract table data from the document' },
                    { tool: 'ocr_document_get', description: 'View core document metadata' },
                ],
            }));
        }
        // No section specified: return manifest (counts only, not full data)
        const manifest = availableSections.map((key) => {
            const val = extras[key];
            const entry = { name: key };
            if (Array.isArray(val)) {
                entry.type = 'array';
                entry.count = val.length;
            }
            else if (val && typeof val === 'object') {
                entry.type = 'object';
            }
            else {
                entry.type = typeof val;
            }
            return entry;
        });
        return formatResponse(successResult({
            document_id: input.document_id,
            file_name: doc.file_name,
            sections: manifest,
            available_sections: availableSections,
            next_steps: [
                { tool: 'ocr_document_extras', description: "Get a specific section: section='charts'" },
                { tool: 'ocr_document_tables', description: 'Extract table data from the document' },
                { tool: 'ocr_document_get', description: 'View core document metadata' },
            ],
        }));
    }
    catch (error) {
        return handleError(error);
    }
}
/**
 * Build a 2D grid from a parsed table's cells with computed max row/column indices.
 */
function buildTableGrid(table) {
    const rowMap = new Map();
    for (const cell of table.cells) {
        if (!rowMap.has(cell.row))
            rowMap.set(cell.row, new Map());
        rowMap.get(cell.row).set(cell.col, cell.text);
    }
    const maxRow = table.row_count > 0 ? table.row_count - 1 : Math.max(0, ...table.cells.map((c) => c.row));
    const maxCol = table.column_count > 0 ? table.column_count - 1 : Math.max(0, ...table.cells.map((c) => c.col));
    return { rowMap, maxRow, maxCol };
}
// ═══════════════════════════════════════════════════════════════════════════════
// INPUT SCHEMA: ocr_table_export
// ═══════════════════════════════════════════════════════════════════════════════
const TableExportInput = z.object({
    document_id: z.string().min(1).describe('Document ID to export tables from'),
    table_index: z
        .number()
        .int()
        .min(0)
        .optional()
        .describe('Specific table index (0-based). Omit to export all tables.'),
    format: z
        .enum(['csv', 'json', 'markdown'])
        .default('json')
        .describe('Export format: csv (RFC 4180), json (structured), or markdown (pipe-delimited)'),
});
// ═══════════════════════════════════════════════════════════════════════════════
// HANDLER: ocr_table_export
// ═══════════════════════════════════════════════════════════════════════════════
/**
 * Handle ocr_table_export - Export table data in CSV, JSON, or markdown format
 */
async function handleTableExport(params) {
    try {
        const input = validateInput(TableExportInput, params);
        const { db } = requireDatabase();
        const doc = db.getDocument(input.document_id);
        if (!doc) {
            throw documentNotFoundError(input.document_id);
        }
        const nextSteps = [
            { tool: 'ocr_document_tables', description: 'View table structure and cell data' },
            { tool: 'ocr_search', description: 'Search for related content' },
        ];
        // Try json_blocks first, then fall back to chunk-based table parsing
        const blocksResult = fetchJsonBlocks(db.getConnection(), input.document_id);
        let allTables;
        if (blocksResult.ok) {
            allTables = extractTablesFromBlocks(blocksResult.blocks);
        }
        else {
            // Fallback: extract tables from chunk text (markdown pipe-delimited tables)
            console.error(`[TableExport] json_blocks unavailable (${blocksResult.reason}) for document ${input.document_id}, falling back to chunk-based table extraction`);
            allTables = parseTablesFromChunks(db.getConnection(), input.document_id, 'TableExport');
        }
        if (allTables.length === 0) {
            throw new MCPError('VALIDATION_ERROR', `No tables found in document ${input.document_id}. The document has no table content.`);
        }
        const tables = filterTablesByIndex(allTables, input.table_index);
        if (tables === null) {
            return formatResponse(successResult({
                document_id: input.document_id,
                file_name: doc.file_name,
                total_tables: allTables.length,
                requested_index: input.table_index,
                format: input.format,
                message: `Table index ${input.table_index} out of range. Document has ${allTables.length} table(s).`,
                next_steps: nextSteps,
            }));
        }
        // Format output based on requested format
        if (input.format === 'csv') {
            const csvQuote = (value) => `"${value.replace(/"/g, '""')}"`;
            const csvParts = [];
            for (const table of tables) {
                if (table.cells.length === 0)
                    continue;
                const { rowMap, maxRow, maxCol } = buildTableGrid(table);
                const lines = [];
                for (let r = 0; r <= maxRow; r++) {
                    const row = rowMap.get(r);
                    const cols = [];
                    for (let c = 0; c <= maxCol; c++) {
                        cols.push(csvQuote(row?.get(c) ?? ''));
                    }
                    lines.push(cols.join(','));
                }
                csvParts.push(lines.join('\n'));
            }
            return formatResponse(successResult({
                document_id: input.document_id,
                file_name: doc.file_name,
                total_tables: allTables.length,
                exported_tables: tables.length,
                format: 'csv',
                data: csvParts.join('\n\n'),
                next_steps: nextSteps,
            }));
        }
        if (input.format === 'markdown') {
            const mdParts = [];
            for (const table of tables) {
                if (table.cells.length === 0)
                    continue;
                const { rowMap, maxRow, maxCol } = buildTableGrid(table);
                const lines = [];
                // Header row
                const headerRow = rowMap.get(0);
                const headerCells = [];
                for (let c = 0; c <= maxCol; c++) {
                    headerCells.push(headerRow?.get(c) ?? '');
                }
                lines.push(`| ${headerCells.join(' | ')} |`);
                lines.push(`| ${headerCells.map(() => '---').join(' | ')} |`);
                // Data rows
                for (let r = 1; r <= maxRow; r++) {
                    const row = rowMap.get(r);
                    const cells = [];
                    for (let c = 0; c <= maxCol; c++) {
                        cells.push(row?.get(c) ?? '');
                    }
                    lines.push(`| ${cells.join(' | ')} |`);
                }
                if (table.caption) {
                    lines.unshift(`**${table.caption}**`);
                }
                mdParts.push(lines.join('\n'));
            }
            return formatResponse(successResult({
                document_id: input.document_id,
                file_name: doc.file_name,
                total_tables: allTables.length,
                exported_tables: tables.length,
                format: 'markdown',
                data: mdParts.join('\n\n'),
                next_steps: nextSteps,
            }));
        }
        // Default: JSON format
        const jsonTables = tables.map((t) => {
            const { rowMap, maxRow } = buildTableGrid(t);
            // Build column names from first row
            const headerRow = rowMap.get(0);
            const colNames = [];
            if (headerRow) {
                for (const [col, text] of headerRow) {
                    colNames[col] = text;
                }
            }
            // Build data rows as column-keyed objects
            const rows = [];
            for (let r = 1; r <= maxRow; r++) {
                const row = rowMap.get(r);
                const rowObj = {};
                if (row) {
                    for (const [col, text] of row) {
                        const colName = colNames[col] ?? `col_${col}`;
                        rowObj[colName] = text;
                    }
                }
                rows.push(rowObj);
            }
            return {
                table_index: t.table_index,
                page_number: t.page_number,
                caption: t.caption,
                columns: colNames.filter(Boolean),
                row_count: rows.length,
                rows,
            };
        });
        return formatResponse(successResult({
            document_id: input.document_id,
            file_name: doc.file_name,
            total_tables: allTables.length,
            exported_tables: tables.length,
            format: 'json',
            tables: jsonTables,
            next_steps: nextSteps,
        }));
    }
    catch (error) {
        return handleError(error);
    }
}
// ═══════════════════════════════════════════════════════════════════════════════
// TOOL DEFINITIONS EXPORT
// ═══════════════════════════════════════════════════════════════════════════════
/**
 * Intelligence tools collection for MCP server registration
 */
export const intelligenceTools = {
    ocr_guide: {
        description: '[ESSENTIAL] System state overview with prioritized next_steps. Shows databases, stats, and tool recommendations. Optional intent: explore/search/ingest/analyze/status.',
        inputSchema: GuideInput.shape,
        handler: handleGuide,
    },
    ocr_document_tables: {
        description: '[ANALYSIS] Extract table data from a document. Returns table summaries by default (caption, row_count, column_count). Use include_cells=true for cell data. Paginated (default 10).',
        inputSchema: DocumentTablesInput.shape,
        handler: handleDocumentTables,
    },
    ocr_document_recommend: {
        description: '[ANALYSIS] Related document recommendations via cluster membership and vector similarity. Requires embeddings and/or clustering.',
        inputSchema: DocumentRecommendInput.shape,
        handler: handleDocumentRecommend,
    },
    ocr_document_extras: {
        description: '[ANALYSIS] Supplementary OCR data. Without section param, returns a manifest with counts per section. Specify section for data (paginated for arrays).',
        inputSchema: DocumentExtrasInput.shape,
        handler: handleDocumentExtras,
    },
    ocr_table_export: {
        description: '[ANALYSIS] Export table data as CSV, JSON, or markdown. Specify table_index for one table, or omit for all. JSON format returns rows with column-keyed objects.',
        inputSchema: TableExportInput.shape,
        handler: handleTableExport,
    },
};
//# sourceMappingURL=intelligence.js.map