// lib/tools/adapters/mines/document-extract.js
//
// mines_document_extract — parse a source document, OCR if scanned, chunk the
// text, embed the chunks, and write mines_document_chunks rows.
//
// Parsers (pdf-parse / mammoth / officeparser) and the embedder are OPTIONAL
// runtime dependencies loaded via bundler-ignored dynamic imports, so the
// Next.js build never tries to resolve them. Graceful degradation:
//   - parser missing            → extraction_status = 'failed' (parser_unavailable)
//   - scanned PDF, no OCR engine → extraction_status = 'ocr_skipped'
//   - chunks created, no embedder→ extraction_status = 'embedding_failed' (RAG
//                                   falls back to text search; numbers stay in SQL)
//   - success                    → extraction_status = 'complete'
//
// Risk: medium. No external network calls (embedding model is local).

import { getDb, newId, nowIso } from '../../../db.js';
import { embedChunks, serializeEmbedding } from '../../../mines/embeddings.js';
import { isScannedPdf, ocrPdf, SCANNED_PDF_THRESHOLD } from '../../../mines/ocr.js';
import { createLocalStorage } from '../../../mines/storage.js';

const TARGET_TOKENS = Number(process.env.MINES_CHUNK_TOKENS) || 600;
const OVERLAP_TOKENS = Number(process.env.MINES_CHUNK_OVERLAP_TOKENS) || 100;

// ── optional parsers ──────────────────────────────────────────────────────────

async function optionalImport(name) {
  try {
    switch (name) {
      case 'pdf-parse':
        return await import(/* webpackIgnore: true */ /* turbopackIgnore: true */ 'pdf-parse');
      case 'mammoth':
        return await import(/* webpackIgnore: true */ /* turbopackIgnore: true */ 'mammoth');
      case 'officeparser':
        return await import(/* webpackIgnore: true */ /* turbopackIgnore: true */ 'officeparser');
      default:
        return null;
    }
  } catch {
    return null;
  }
}

// ── text helpers ────────────────────────────────────────────────────────────

function estimateTokens(text) {
  return Math.ceil(String(text || '').length / 4);
}

const ES_HINTS = ['de', 'la', 'el', 'que', 'en', 'los', 'para', 'con', 'del', 'una', 'por', 'se', 'su'];
const EN_HINTS = ['the', 'of', 'and', 'to', 'in', 'for', 'is', 'that', 'with', 'from', 'this', 'are'];

/** Lightweight Spanish/English detector. Defaults to 'es' (Spanish-first corpus). */
function detectLanguage(text) {
  const tokens = String(text || '').toLowerCase().split(/[^a-záéíóúñ]+/i).filter(Boolean).slice(0, 400);
  if (tokens.length < 5) return 'es';
  let es = 0;
  let en = 0;
  for (const t of tokens) {
    if (ES_HINTS.includes(t)) es++;
    if (EN_HINTS.includes(t)) en++;
  }
  return en > es * 1.2 ? 'en' : 'es';
}

/**
 * Split one page of text into ≤TARGET_TOKENS chunks with OVERLAP_TOKENS overlap.
 * Boundaries snap to sentence / paragraph ends.
 */
function chunkPageText(text, page) {
  const targetChars = TARGET_TOKENS * 4;
  const overlapChars = OVERLAP_TOKENS * 4;
  const clean = String(text || '').replace(/\r/g, '').replace(/[ \t]+/g, ' ').trim();
  if (!clean) return [];

  const sentences = clean.split(/(?<=[.!?。])\s+|\n{2,}/).map(s => s.trim()).filter(Boolean);
  const chunks = [];
  let buf = '';
  for (const s of sentences) {
    if (buf && buf.length + 1 + s.length > targetChars) {
      chunks.push(buf.trim());
      const tail = buf.slice(Math.max(0, buf.length - overlapChars));
      buf = `${tail} ${s}`;
    } else {
      buf = buf ? `${buf} ${s}` : s;
    }
  }
  if (buf.trim()) chunks.push(buf.trim());
  return chunks.map(t => ({ text: t, page, token_count: estimateTokens(t) }));
}

// ── extraction by file type ─────────────────────────────────────────────────

/**
 * @returns {{ available, pages: Array<{page,text}>, pageCount, ocrApplied,
 *             isScanned, ocrSkipped, error? }}
 */
async function extractDocument(filePath, fileType, forceOcr, store) {
  if (fileType === 'pdf') {
    const mod = await optionalImport('pdf-parse');
    // Support both pdf-parse v1 (default export is a function) and v2 (named
    // PDFParse class with a getText() method returning { pages, text, total }).
    const pdfParseFn = typeof mod === 'function' ? mod : (typeof mod?.default === 'function' ? mod.default : null);
    const PDFParseClass = mod?.PDFParse || mod?.default?.PDFParse || null;
    if (!pdfParseFn && !PDFParseClass) return { available: false, error: 'parser_unavailable:pdf', pages: [] };

    const buf = store.readDocument(filePath);
    let fullText = '';
    let pageCount = 1;
    let perPage = null; // [{ page, text }] when the parser exposes per-page text
    try {
      if (PDFParseClass) {
        const parser = new PDFParseClass({ data: buf });
        const res = await parser.getText();
        try { parser.destroy?.(); } catch { /* ignore */ }
        fullText = String(res?.text || '');
        pageCount = Number(res?.total) || (Array.isArray(res?.pages) ? res.pages.length : 1) || 1;
        if (Array.isArray(res?.pages)) {
          perPage = res.pages.map((p, i) => ({ page: Number(p?.num ?? p?.page ?? i + 1), text: String(p?.text || '') }));
        }
      } else {
        const data = await pdfParseFn(buf);
        fullText = String(data?.text || '');
        pageCount = Number(data?.numpages) || 1;
      }
    } catch (err) {
      return { available: true, error: 'parse_error:' + (err?.message || 'pdf'), pages: [] };
    }
    const avgChars = fullText.length / pageCount;

    if (forceOcr || isScannedPdf(fullText, avgChars)) {
      const ocr = await ocrPdf(filePath);
      if (ocr.available) {
        return {
          available: true,
          isScanned: true,
          ocrApplied: true,
          pageCount: ocr.pages.length || pageCount,
          pages: ocr.pages.map(p => ({ page: p.page, text: p.text })),
        };
      }
      return { available: true, isScanned: true, ocrApplied: false, ocrSkipped: true, pageCount, pages: [] };
    }

    // Prefer real per-page text (accurate page citations); else split on \f.
    if (perPage) {
      return { available: true, isScanned: false, ocrApplied: false, pageCount, pages: perPage };
    }
    const pageTexts = fullText.includes('\f') ? fullText.split('\f') : [fullText];
    return {
      available: true,
      isScanned: false,
      ocrApplied: false,
      pageCount,
      pages: pageTexts.map((text, i) => ({ page: i + 1, text })),
    };
  }

  if (fileType === 'docx') {
    const mod = await optionalImport('mammoth');
    const extract = mod?.extractRawText || mod?.default?.extractRawText;
    if (typeof extract !== 'function') return { available: false, error: 'parser_unavailable:docx', pages: [] };
    try {
      const result = await extract({ path: filePath });
      return { available: true, isScanned: false, ocrApplied: false, pageCount: 1, pages: [{ page: 1, text: result?.value || '' }] };
    } catch (err) {
      return { available: true, error: 'parse_error:' + (err?.message || 'docx'), pages: [] };
    }
  }

  if (fileType === 'pptx') {
    const mod = await optionalImport('officeparser');
    // officeparser v4 exposed parseOfficeAsync; v5 exposes parseOffice (Promise).
    const parse = mod?.parseOfficeAsync || mod?.default?.parseOfficeAsync
      || mod?.parseOffice || mod?.default?.parseOffice;
    if (typeof parse !== 'function') return { available: false, error: 'parser_unavailable:pptx', pages: [] };
    try {
      const text = await parse(filePath);
      return { available: true, isScanned: false, ocrApplied: false, pageCount: 1, pages: [{ page: 1, text: String(text || '') }] };
    } catch (err) {
      return { available: true, error: 'parse_error:' + (err?.message || 'pptx'), pages: [] };
    }
  }

  return { available: false, error: 'unsupported_file_type:' + fileType, pages: [] };
}

// ── adapter ───────────────────────────────────────────────────────────────────

/**
 * @param {object} input - { source_document_id, force_ocr?, approval_id? }
 * @returns {Promise<object>} { status, chunks_created, pages_processed, language_detected, ocr_applied }
 */
export async function minesDocumentExtract(input, options = {}) {
  const sourceDocumentId = input?.source_document_id;
  if (!sourceDocumentId) return { status: 'error', error: 'source_document_id_required' };

  const db = getDb();
  const doc = db.prepare(
    'SELECT id, project_id, document_category, original_path, file_type FROM mines_source_documents WHERE id = ?'
  ).get(sourceDocumentId);
  if (!doc) return { status: 'error', error: 'source_document_not_found' };

  const setStatus = db.prepare(`
    UPDATE mines_source_documents
    SET extraction_status = ?, extraction_error = ?, language = ?, is_scanned = ?,
        ocr_applied = ?, page_count = ?, avg_chars_per_page = ?, updated_at = ?
    WHERE id = ?
  `);

  // Mark in-flight so the Data tab progress bar reflects active work.
  db.prepare("UPDATE mines_source_documents SET extraction_status = 'extracting', updated_at = ? WHERE id = ?")
    .run(nowIso(), doc.id);

  // Storage interface (local-disk impl). Reads route through the dataroom
  // storage layer, which re-enforces the import-root allowlist (org-scoped).
  const store = createLocalStorage({ org_id: options.org_id });

  const extracted = await extractDocument(doc.original_path, doc.file_type, Boolean(input?.force_ocr), store);

  if (!extracted.available || extracted.error) {
    const status = extracted.ocrSkipped ? 'ocr_skipped' : 'failed';
    setStatus.run(
      status, extracted.error || (extracted.ocrSkipped ? 'no_ocr_engine' : 'extract_failed'),
      'es', extracted.isScanned ? 1 : 0, 0, extracted.pageCount ?? null, null, nowIso(), doc.id
    );
    return { status, chunks_created: 0, pages_processed: 0, language_detected: 'es', ocr_applied: false, reason: extracted.error };
  }

  if (extracted.ocrSkipped) {
    setStatus.run('ocr_skipped', 'scanned_pdf_no_ocr_engine', 'es', 1, 0, extracted.pageCount ?? null, null, nowIso(), doc.id);
    return { status: 'ocr_skipped', chunks_created: 0, pages_processed: 0, language_detected: 'es', ocr_applied: false };
  }

  // Build chunks across pages.
  const chunks = [];
  let totalChars = 0;
  let pagesWithText = 0;
  for (const p of extracted.pages) {
    const pageChunks = chunkPageText(p.text, p.page);
    if (pageChunks.length) pagesWithText++;
    totalChars += String(p.text || '').length;
    for (const c of pageChunks) {
      c.language = detectLanguage(c.text);
      chunks.push(c);
    }
  }

  const pageCount = extracted.pageCount || extracted.pages.length || 1;
  const avgChars = pageCount ? totalChars / pageCount : totalChars;
  const docLanguage = chunks.length ? detectLanguage(chunks.map(c => c.text).join(' ').slice(0, 4000)) : 'es';

  if (chunks.length === 0) {
    // Parsed but no extractable text — treat as scanned/empty and remove any
    // stale chunks from a previous retry of the same source document.
    db.prepare('DELETE FROM mines_document_chunks WHERE source_document_id = ?').run(doc.id);
    const scanned = avgChars < SCANNED_PDF_THRESHOLD;
    setStatus.run(scanned ? 'ocr_skipped' : 'complete', scanned ? 'no_text_extracted' : null,
      docLanguage, scanned ? 1 : 0, extracted.ocrApplied ? 1 : 0, pageCount, avgChars, nowIso(), doc.id);
    return {
      status: scanned ? 'ocr_skipped' : 'complete',
      chunks_created: 0, pages_processed: pagesWithText, language_detected: docLanguage,
      ocr_applied: Boolean(extracted.ocrApplied),
    };
  }

  // Embed (best-effort).
  const { embedded, available: embAvailable } = await embedChunks(chunks);

  // Insert chunks.
  const insertChunk = db.prepare(`
    INSERT INTO mines_document_chunks
      (id, source_document_id, project_id, document_category, page, chunk_index,
       text, language, embedding, token_count, created_at)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    ON CONFLICT(source_document_id, chunk_index) DO UPDATE SET
      project_id = excluded.project_id,
      document_category = excluded.document_category,
      page = excluded.page,
      text = excluded.text,
      language = excluded.language,
      embedding = excluded.embedding,
      token_count = excluded.token_count,
      created_at = excluded.created_at
  `);
  const now = nowIso();
  db.prepare('BEGIN').run();
  try {
    db.prepare('DELETE FROM mines_document_chunks WHERE source_document_id = ? AND chunk_index >= ?')
      .run(doc.id, chunks.length);
    chunks.forEach((c, idx) => {
      insertChunk.run(
        newId('mdc'), doc.id, doc.project_id, doc.document_category,
        c.page ?? null, idx, c.text, c.language || docLanguage,
        serializeEmbedding(c.embedding), c.token_count ?? estimateTokens(c.text), now
      );
    });
    db.prepare('COMMIT').run();
  } catch (err) {
    try { db.prepare('ROLLBACK').run(); } catch {}
    setStatus.run('failed', 'chunk_insert_error:' + (err?.message || ''), docLanguage,
      extracted.isScanned ? 1 : 0, extracted.ocrApplied ? 1 : 0, pageCount, avgChars, nowIso(), doc.id);
    throw err;
  }

  const finalStatus = embAvailable ? 'complete' : 'embedding_failed';
  setStatus.run(
    finalStatus, embAvailable ? null : 'embedding_provider_unavailable',
    docLanguage, extracted.isScanned ? 1 : 0, extracted.ocrApplied ? 1 : 0,
    pageCount, avgChars, nowIso(), doc.id
  );

  return {
    status: finalStatus,
    chunks_created: chunks.length,
    chunks_embedded: embedded,
    pages_processed: pagesWithText,
    language_detected: docLanguage,
    ocr_applied: Boolean(extracted.ocrApplied),
  };
}
