// lib/tools/adapters/mines/data-import.js
//
// mines_data_import — bulk-import a mine data directory tree.
//
// Walks a directory recursively (node:fs only — no shell), classifies each file
// by the 23-category folder taxonomy, and routes it:
//   .pdf/.docx/.pptx           → mines.extract_document  heartbeat task
//   .xlsx/.xls/.csv            → mines.load_tabular       heartbeat task
//   .shp/.gdb/.mxd/.dxf/.dwg/  → mines.index_spatial      heartbeat task
//     .dbf/.shx/.prj/.adf/.raster
//   .jpg/.jpeg/.png/.tif/.tiff → catalog_only (row only, no extraction)
//   everything else            → unsupported (row only)
//
// Idempotent + resumable on (project_id, original_path): re-running never
// duplicates rows. Source-document inserts run in batches of 100 inside a
// transaction; heartbeat tasks are enqueued after each batch commits.
//
// Risk: high (approval-gated at the registry; the per-file processing tasks run
// inside the already-approved import and are dispatched directly by the worker).

import path from 'node:path';
import { getDb, newId, nowIso } from '../../../db.js';
import { isInsideAllowedRoots } from '../../safety.js';
import { isInsideApprovedMinesImportRoot } from '../../../mines/import-roots.js';
import { createLocalStorage } from '../../../mines/storage.js';
import { getMinesProjectForOrg } from '../../../mines/project-auth.js';
import { classifyByFolderPath } from '../../../mines/taxonomy.js';
import { createHeartbeatTask } from '../../../heartbeat-queue.js';
import { appendComplianceAudit } from '../../../compliance-audit.js';

const DOC_EXTS = new Set(['.pdf', '.docx', '.pptx']);
const TABULAR_EXTS = new Set(['.xlsx', '.xls', '.csv']);
const SPATIAL_EXTS = new Set(['.shp', '.gdb', '.mxd', '.dxf', '.dwg', '.dbf', '.shx', '.prj', '.adf', '.raster']);
const PHOTO_EXTS = new Set(['.jpg', '.jpeg', '.png', '.tif', '.tiff']);

const BATCH_SIZE = 100;

function fileTypeForExt(ext) {
  const e = ext.replace(/^\./, '').toLowerCase();
  return e || 'other';
}

function routeForExt(ext) {
  if (DOC_EXTS.has(ext)) return { kind: 'document', task: 'mines.extract_document', status: 'queued' };
  if (TABULAR_EXTS.has(ext)) return { kind: 'tabular', task: 'mines.load_tabular', status: 'queued' };
  if (SPATIAL_EXTS.has(ext)) return { kind: 'spatial', task: 'mines.index_spatial', status: 'queued' };
  if (PHOTO_EXTS.has(ext)) return { kind: 'photo', task: null, status: 'catalog_only' };
  return { kind: 'unsupported', task: null, status: 'unsupported' };
}

/**
 * @param {object} input
 * @param {string} input.project_id        mines_projects.id
 * @param {string} input.directory_root    Directory to import (alias: directory_path)
 * @param {boolean} [input.resume]         Skip files already past 'queued'.
 * @param {object} [options]
 * @returns {Promise<{ status, files_found, queued, catalog_only, unsupported, skipped_complete, errors: string[] }>}
 */
export async function minesDataImport(input, options = {}) {
  const project_id = input?.project_id;
  const directory_root = input?.directory_root || input?.directory_path;
  const resume = Boolean(input?.resume);
  const errors = [];

  if (!project_id) return { status: 'error', error: 'project_id_required' };
  if (!directory_root) return { status: 'error', error: 'directory_root_required' };

  // Gate 1: allowlist containment — BEFORE any DB write. Import roots may be
  // approved persistently from the UI, so operators do not have to edit
  // .env.local for every dataset.
  if (!isInsideAllowedRoots(directory_root) && !isInsideApprovedMinesImportRoot(directory_root, { org_id: options.org_id })) {
    throw new Error('allowlist_violation: directory_root is outside the OS-allowed or approved Mines import roots: ' + directory_root);
  }

  // Storage interface (local-disk impl). Defence-in-depth on top of the entry
  // allowlist check above: every listFolder/statFile call re-enforces the
  // import-root allowlist, org-scoped via options.org_id.
  const store = createLocalStorage({ org_id: options.org_id });

  const absRoot = path.resolve(directory_root);
  const rootStat = store.statFile(absRoot);
  if (!rootStat) {
    return { status: 'error', error: 'directory_not_found', errors: ['Directory does not exist: ' + absRoot] };
  }
  if (!rootStat.isDirectory) {
    return { status: 'error', error: 'not_a_directory', errors: ['Path is not a directory: ' + absRoot] };
  }

  const db = getDb();

  // Confirm the project exists (FK target) and, when called from an org-scoped
  // surface, belongs to the caller's org. Avoids cross-org import writes.
  const projectRow = options.org_id
    ? getMinesProjectForOrg(project_id, options.org_id)
    : db.prepare('SELECT id FROM mines_projects WHERE id = ?').get(project_id);
  if (!projectRow) return { status: 'error', error: 'project_not_found' };

  const files = store.listFolder(absRoot);

  const findExisting = db.prepare(
    'SELECT id, extraction_status FROM mines_source_documents WHERE project_id = ? AND original_path = ? LIMIT 1'
  );
  const insertDoc = db.prepare(`
    INSERT OR IGNORE INTO mines_source_documents
      (id, project_id, document_category, original_path, file_name, file_type,
       file_size_bytes, sha256, extraction_status, language, ingested_by_agent_id,
       created_at, updated_at)
    VALUES (?, ?, ?, ?, ?, ?, ?, NULL, ?, 'es', ?, ?, ?)
  `);

  let files_found = 0;
  let queued = 0;
  let catalog_only = 0;
  let unsupported = 0;
  let skipped_complete = 0;
  const agentId = options.agent_id || null;

  // Process in batches: insert source-document rows in a transaction, then
  // enqueue heartbeat tasks for the newly-inserted processing files after the
  // commit (createHeartbeatTask runs its own statements outside this txn).
  for (let i = 0; i < files.length; i += BATCH_SIZE) {
    const batch = files.slice(i, i + BATCH_SIZE);
    const toEnqueue = [];

    db.prepare('BEGIN').run();
    try {
      for (const full of batch) {
        files_found++;
        const ext = path.extname(full).toLowerCase();
        const route = routeForExt(ext);
        const category = classifyByFolderPath(full);
        const fileName = path.basename(full);

        let size = null;
        try { size = store.statFile(full)?.size ?? null; } catch { /* ignore */ }

        const existing = findExisting.get(project_id, full);
        if (existing) {
          // Incremental + resumable: never duplicate an existing path. Terminal
          // rows are done; non-terminal rows are already queued/in-flight. Either
          // way we skip and never enqueue a second task for the same file.
          skipped_complete++;
          continue;
        }

        const now = nowIso();
        const id = newId('msd');
        const res = insertDoc.run(
          id, project_id, category, full, fileName,
          fileTypeForExt(ext), size, route.status, agentId, now, now
        );
        if (res.changes === 0) {
          // Unique index closed a concurrent SELECT→INSERT race.
          skipped_complete++;
          continue;
        }

        if (route.kind === 'document' || route.kind === 'tabular' || route.kind === 'spatial') {
          queued++;
          toEnqueue.push({ task: route.task, source_document_id: id });
        } else if (route.kind === 'photo') {
          catalog_only++;
        } else {
          unsupported++;
        }
      }
      db.prepare('COMMIT').run();
    } catch (err) {
      try { db.prepare('ROLLBACK').run(); } catch {}
      errors.push('Batch insert failed at offset ' + i + ': ' + (err?.message || String(err)));
      continue; // skip enqueue for the failed batch
    }

    for (const t of toEnqueue) {
      try {
        createHeartbeatTask({
          type: t.task,
          agent_id: agentId,
          project_id,
          payload: { type: t.task, source_document_id: t.source_document_id, project_id },
        });
      } catch (err) {
        errors.push('Failed to enqueue ' + t.task + ' for ' + t.source_document_id + ': ' + (err?.message || String(err)));
      }
    }
  }

  // Completion audit (non-fatal on failure).
  try {
    appendComplianceAudit({
      user_id: options.acting_on_behalf_of_user_id || options.user_id || 'system',
      org_id: options.org_id || 'personal-org',
      action: 'mines.data_import.completed',
      status: 'success',
      summary:
        'Imported ' + files_found + ' files for project ' + project_id +
        ' — ' + queued + ' queued, ' + catalog_only + ' catalog-only, ' +
        unsupported + ' unsupported, ' + skipped_complete + ' skipped',
      action_target_id: project_id,
      action_target_type: 'mines_project',
      meta: { project_id, directory_root: absRoot, files_found, queued, catalog_only, unsupported, skipped_complete, resume },
    });
  } catch { /* audit failure is non-fatal */ }

  return { status: 'ok', files_found, queued, catalog_only, unsupported, skipped_complete, errors };
}
