/**
 * src/rag/ingest.ts
 * استيعاب المستندات — Document ingestion for RAG pipeline
 */
import { readFile } from 'node:fs/promises';
import { supabase } from '../utils/supabase.js';
import { createChildLogger } from '../utils/logger.js';
const log = createChildLogger('rag:ingest');
// ─── Document Ingestion ─────────────────────────────────────────────
export async function ingestDocument(tenantId, filePath, fileType) {
    log.info({ tenantId, filePath, fileType }, 'Starting document ingestion');
    // 1. Extract text based on file type
    let text;
    try {
        text = await extractText(filePath, fileType);
    }
    catch (err) {
        log.error({ filePath, fileType, err }, 'Failed to extract text');
        throw new Error(`فشل في استخراج النص من الملف: ${err.message}`);
    }
    if (!text.trim()) {
        throw new Error('الملف لا يحتوي على نص قابل للقراءة');
    }
    // 2. Create document record
    const { data: doc, error: docErr } = await supabase
        .from('documents')
        .insert({
        tenant_id: tenantId,
        file_path: filePath,
        file_type: fileType,
        status: 'processing',
        total_characters: text.length,
    })
        .select('id')
        .single();
    if (docErr || !doc) {
        throw new Error(`فشل في إنشاء سجل المستند: ${docErr?.message}`);
    }
    const documentId = doc.id;
    try {
        // 3. Chunk text
        const chunks = chunkText(text, 1000);
        log.info({ documentId, chunkCount: chunks.length }, 'Text chunked');
        // 4. Build chunk metadata
        const docChunks = chunks.map((content, index) => ({
            content,
            index,
            metadata: {
                source: filePath,
                fileType,
                chunkIndex: index,
                totalChunks: chunks.length,
            },
        }));
        // 5. Generate embeddings
        const embeddings = await embedChunks(chunks);
        // 6. Store chunks
        await storeChunks(tenantId, documentId, docChunks, embeddings);
        // 7. Update document status
        await supabase
            .from('documents')
            .update({ status: 'ready', chunk_count: chunks.length })
            .eq('id', documentId);
        log.info({ tenantId, documentId, chunks: chunks.length }, 'Document ingested successfully');
        return {
            documentId,
            chunksCreated: chunks.length,
            fileType,
            totalCharacters: text.length,
            extractedText: text,
        };
    }
    catch (err) {
        // Mark as failed
        await supabase
            .from('documents')
            .update({ status: 'failed', error: err.message })
            .eq('id', documentId);
        throw err;
    }
}
/**
 * (User-requested) Ingest a document AND extract a skill from its content if
 * the document describes a workflow / lookup / process worth saving.
 *
 * Pipeline: ingestDocument → if extractable → skill_create via the same path
 * the brain uses (so dedup, KG mirror, and cache-invalidation all happen).
 *
 * Supports: pdf, docx (via 'text' for now), images (OCR), markdown, plain text.
 * Returns the ingest result + the names of any skills auto-created.
 */
export async function learnSkillFromDocument(tenantId, filePath, fileType, options = {}) {
    const ingest = await ingestDocument(tenantId, filePath, fileType);
    const skillsCreated = [];
    const text = ingest.extractedText || '';
    if (text.trim().length < 200) {
        log.debug({ tenantId, filePath, len: text.length }, 'document too short for skill extraction');
        return { ingest, skillsCreated };
    }
    try {
        const { callWithFailover } = await import('../models/failover.js');
        const { executeTool } = await import('../tools/executor.js');
        const corpus = text.slice(0, 12_000);
        const systemPrompt = `أنت منسّق معرفة. تقرأ مستنداً يقدمه المالك (سياسة، إجراء، عقد، دليل، صورة بنص...) وتقرر:` +
            ` هل هذا يستحق الحفظ كمهارة قابلة لإعادة الاستخدام؟ إذا نعم، اشتق منها مهارة بأقل قدر من التعديل.` +
            ` معايير الإيجاب: workflow، policy، lookup ثابت، إجراء قابل للتكرار، dictionary/glossary صغير.` +
            ` معايير الرفض: مستند مالي مرة واحدة (فاتورة، إيصال)، معرف هوية، رسالة شخصية.\n\n` +
            `أرجع JSON فقط:\n` +
            `{"create": true|false, "name": "...", "whenToUse": "...", "instructions": "...", "examples": []}`;
        const userPrompt = `${options.hint ? `سياق المالك: ${options.hint}\n\n` : ''}محتوى المستند (${ingest.fileType}, ${ingest.totalCharacters} حرف):\n${corpus}`;
        const result = await callWithFailover('reasoning', [
            { role: 'system', content: systemPrompt },
            { role: 'user', content: userPrompt },
        ], { temperature: 0.3, maxTokens: 1500 }, tenantId);
        const raw = result.content || '';
        const tryParse = (s) => {
            try {
                return JSON.parse(s);
            }
            catch {
                return null;
            }
        };
        let parsed = tryParse(raw.trim());
        if (!parsed) {
            const fence = raw.match(/```(?:json)?\s*([\s\S]+?)\s*```/i);
            if (fence)
                parsed = tryParse(fence[1].trim());
        }
        if (!parsed) {
            const m = raw.match(/\{[\s\S]*?"create"[\s\S]*?\}/);
            if (m)
                parsed = tryParse(m[0]);
        }
        if (parsed?.create && parsed.name && parsed.instructions) {
            const r = await executeTool('skill_create', {
                name: parsed.name,
                whenToUse: parsed.whenToUse || `Auto-distilled from document: ${filePath}`,
                instructions: parsed.instructions,
                examples: Array.isArray(parsed.examples) ? parsed.examples : [],
            }, tenantId);
            if (r.success) {
                skillsCreated.push(parsed.name);
                log.info({ tenantId, skill: parsed.name, filePath }, '🌱 auto-created skill from document');
            }
            else {
                log.debug({ tenantId, output: r.output }, 'skill_create from document failed');
            }
        }
    }
    catch (err) {
        log.warn({ tenantId, filePath, err: err.message }, 'document → skill extraction failed (non-fatal)');
    }
    return { ingest, skillsCreated };
}
// ─── Text Extraction ────────────────────────────────────────────────
async function extractText(filePath, fileType) {
    switch (fileType) {
        case 'pdf':
            return extractPdfText(filePath);
        case 'excel':
            return extractExcelText(filePath);
        case 'image':
            return extractImageText(filePath);
        case 'markdown':
        case 'text':
            return readFile(filePath, 'utf-8');
        default:
            throw new Error(`نوع ملف غير مدعوم: ${fileType}`);
    }
}
async function extractPdfText(filePath) {
    try {
        // Dynamic import to avoid requiring pdf-parse when not used
        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
        const pdfParse = await Function('return import("pdf-parse")')().then(m => m.default);
        const buffer = await readFile(filePath);
        const result = await pdfParse(buffer);
        return result.text;
    }
    catch (err) {
        log.warn({ filePath, err }, 'pdf-parse failed, may not be installed');
        throw new Error('مكتبة pdf-parse غير متوفرة. قم بتثبيتها: npm install pdf-parse');
    }
}
async function extractExcelText(filePath) {
    try {
        // Dynamic import — xlsx is optional
        const xlsx = await Function('return import("xlsx")')().then(m => m.default);
        const workbook = xlsx.readFile(filePath);
        const lines = [];
        for (const sheetName of workbook.SheetNames) {
            const sheet = workbook.Sheets[sheetName];
            if (!sheet)
                continue;
            const rows = xlsx.utils.sheet_to_json(sheet, { header: 1 });
            lines.push(`## ${sheetName}`);
            for (const row of rows) {
                const vals = Object.values(row).map(v => String(v ?? '')).filter(Boolean);
                if (vals.length > 0)
                    lines.push(vals.join(' | '));
            }
            lines.push('');
        }
        return lines.join('\n');
    }
    catch (err) {
        log.warn({ filePath, err }, 'xlsx failed, may not be installed');
        throw new Error('مكتبة xlsx غير متوفرة. قم بتثبيتها: npm install xlsx');
    }
}
async function extractImageText(_filePath) {
    // Placeholder: In production, send to vision model for description
    log.warn('Image text extraction requires vision model integration — returning placeholder');
    return '[صورة — يتطلب نموذج رؤية لاستخراج الوصف]';
}
// ─── Arabic-Aware Text Chunking ─────────────────────────────────────
/**
 * Split text into chunks respecting Arabic sentence boundaries.
 * - Splits on sentence-ending punctuation (. ! ؟ 。)
 * - Falls back to paragraph boundaries
 * - Never breaks mid-Arabic-word
 */
export function chunkText(text, maxChunkSize = 1000) {
    if (!text || text.length <= maxChunkSize) {
        return text ? [text.trim()] : [];
    }
    const chunks = [];
    let buffer = '';
    // Split into sentences (Arabic + English punctuation)
    const sentences = text.split(/(?<=[.!?؟。\n\n])\s*/);
    for (const sentence of sentences) {
        const trimmed = sentence.trim();
        if (!trimmed)
            continue;
        if (buffer.length + trimmed.length + 1 <= maxChunkSize) {
            buffer += (buffer ? ' ' : '') + trimmed;
        }
        else {
            // Current buffer is a chunk
            if (buffer)
                chunks.push(buffer.trim());
            // If single sentence exceeds max, split by words
            if (trimmed.length > maxChunkSize) {
                const wordChunks = splitByWords(trimmed, maxChunkSize);
                chunks.push(...wordChunks.slice(0, -1));
                buffer = wordChunks[wordChunks.length - 1] ?? '';
            }
            else {
                buffer = trimmed;
            }
        }
    }
    if (buffer.trim()) {
        chunks.push(buffer.trim());
    }
    return chunks;
}
function splitByWords(text, maxSize) {
    const words = text.split(/\s+/);
    const chunks = [];
    let current = '';
    for (const word of words) {
        if (current.length + word.length + 1 <= maxSize) {
            current += (current ? ' ' : '') + word;
        }
        else {
            if (current)
                chunks.push(current);
            current = word;
        }
    }
    if (current)
        chunks.push(current);
    return chunks;
}
// ─── Embedding Generation ───────────────────────────────────────────
/**
 * Generate embeddings for text chunks.
 * Uses OpenAI-compatible embedding API.
 * Placeholder — replace endpoint/model as needed.
 */
export async function embedChunks(chunks) {
    const apiKey = process.env['OPENAI_API_KEY'];
    if (!apiKey) {
        log.warn('No OPENAI_API_KEY set, returning zero embeddings (placeholder)');
        return chunks.map(() => new Array(1536).fill(0));
    }
    try {
        const response = await fetch('https://api.openai.com/v1/embeddings', {
            method: 'POST',
            headers: {
                'Authorization': `Bearer ${apiKey}`,
                'Content-Type': 'application/json',
            },
            body: JSON.stringify({
                model: 'text-embedding-3-small',
                input: chunks,
            }),
            signal: AbortSignal.timeout(30_000),
        });
        if (!response.ok) {
            throw new Error(`Embedding API returned ${response.status}`);
        }
        const json = await response.json();
        // Sort by index to maintain order
        return json.data
            .sort((a, b) => a.index - b.index)
            .map(d => d.embedding);
    }
    catch (err) {
        log.error({ err }, 'Failed to generate embeddings');
        // Return zero vectors as fallback
        return chunks.map(() => new Array(1536).fill(0));
    }
}
// ─── Store Chunks in Supabase ───────────────────────────────────────
export async function storeChunks(tenantId, documentId, chunks, embeddings) {
    // chunk_index isn't a column on document_chunks (both Postgres and SQLite
    // schemas drop it). Stash it inside metadata.chunk_index instead so callers
    // that need ordering can still recover it.
    const rows = chunks.map((chunk, i) => ({
        tenant_id: tenantId,
        document_id: documentId,
        content: chunk.content,
        embedding: embeddings[i] ?? [],
        metadata: { ...chunk.metadata, chunk_index: chunk.index },
    }));
    // Batch insert (Supabase handles up to ~1000 rows)
    const batchSize = 100;
    for (let i = 0; i < rows.length; i += batchSize) {
        const batch = rows.slice(i, i + batchSize);
        const { error } = await supabase
            .from('document_chunks')
            .insert(batch);
        if (error) {
            log.error({ documentId, batch: i, error: error.message }, 'Failed to store chunk batch');
            throw new Error(`فشل في حفظ القطع: ${error.message}`);
        }
    }
    log.info({ documentId, totalChunks: chunks.length }, 'Chunks stored successfully');
}
//# sourceMappingURL=ingest.js.map