// lib/mines/tabular-loaders.js
//
// Tabular ingestion for the Mines module: assay-sample and drill-hole loaders.
//
// Handles real-world messiness from the Batamote corpus:
//  - "Ensayes" junk-header files (2–4 metadata rows before the real header)
//  - Spanish + English column names (Muestra/Sample, Este/Easting, Au/Au_ppm…)
//  - Comma decimal separators (Spanish locale)
//  - Blank cells → NULL in DB (never coerced to 0)
//
// .xlsx parsed with exceljs (already a dependency). Legacy binary .xls parsed
// with SheetJS (xlsx) via an optional dynamic import — exceljs only reads the
// zip-based .xlsx format and throws on OLE-compound .xls. CSV parsed with a
// small dependency-free reader. See engineering-design §4 (Tabular path) and §F.

import path from 'node:path';
import fs from 'node:fs';
import ExcelJS from 'exceljs';
import { getDb, newId, nowIso } from '../db.js';

// ── Column alias maps (broad — used for mapping once the type is known) ───────
// Aliases are stored in normalised form (lowercase, non-alphanumeric → single
// space, trimmed). Spanish + English. Matching also accepts a single short
// alias token appearing standalone in a header (so "Au (ppm)" matches "au").

const ASSAY_ALIASES = {
  sample_id:   ['sample', 'muestra', 'sample id', 'sampleid', 'sample no', 'muestra id', 'id muestra', 'id',
                'tbldhsample sampleid', 'tbldhassay01 sampleid'],
  easting:     ['easting', 'este', 'east', 'utm e', 'utm este', 'coordsorigx', 'coordsprefx'],
  northing:    ['northing', 'norte', 'north', 'utm n', 'utm norte', 'coordsorigy', 'coordsprefy'],
  elevation:   ['elevation', 'elev', 'cota', 'altitud', 'elevacion', 'coordsorigz', 'coordsprefz'],
  sample_type: ['sampletype', 'sample type', 'tipo', 'tipo de muestra', 'tipo muestra', 'type'],
  width_m:     ['samplewidth', 'sample width', 'width', 'ancho', 'potencia', 'espesor', 'width m'],
  au_ppm:      ['au ppm', 'au', 'oro', 'au gpt', 'au g t', 'au_ppm', 'aug t'],
  ag_ppm:      ['ag ppm', 'ag', 'plata', 'ag gpt', 'ag g t', 'ag_ppm'],
  // cu_pct holds copper grade as a PERCENT. Some database exports report copper
  // in ppm (e.g. "Cu_ppm"); those columns still map here but the value is
  // converted ppm→pct at load time (see ppmToPct detection in loadAssaySamples).
  cu_pct:      ['cu pct', 'cu', 'cobre', 'cu pc', 'cu_pct', 'cu ppm'],
};

const DRILL_ALIASES = {
  hole_id:          ['holeid', 'hole id', 'hole', 'barreno', 'sondeo', 'pozo', 'dhid', 'dh id', 'bhid'],
  collar_easting:   ['collare', 'collar e', 'collar easting', 'easting', 'east', 'este', 'coordsorigx', 'coordsprefx'],
  collar_northing:  ['collarn', 'collar n', 'collar northing', 'northing', 'north', 'norte', 'coordsorigy', 'coordsprefy'],
  collar_elevation: ['collarelev', 'collar elev', 'collar elevation', 'elevation', 'cota', 'elev', 'rl', 'coordsorigz', 'coordsprefz'],
  azimuth:          ['azimuth', 'azimut', 'azi', 'rumbo', 'bearing', 'azimuthorig', 'azimuthtn', 'azimuthpref'],
  dip:              ['dip', 'inclinacion', 'inclinación', 'buzamiento', 'inclination'],
  total_depth_m:    ['totaldepth', 'total depth', 'depth', 'depthactual', 'mdepth', 'profundidad', 'total depth m', 'prof', 'eoh', 'longitud'],
};

const NUMERIC_FIELDS = new Set([
  'easting', 'northing', 'elevation', 'width_m', 'au_ppm', 'ag_ppm', 'cu_pct',
  'collar_easting', 'collar_northing', 'collar_elevation', 'azimuth', 'dip', 'total_depth_m',
]);

const MAX_HEADER_SCAN = 10;
const MIN_HEADER_MATCHES = 3;

// ── Normalisation + matching ──────────────────────────────────────────────────

function normHeader(raw) {
  return String(raw ?? '')
    .toLowerCase()
    .replace(/[^\p{L}\p{N}]+/gu, ' ')
    .trim();
}

/** Return the field a normalised header maps to, or null. */
function matchField(norm, aliasMap) {
  if (!norm) return null;
  const tokens = norm.split(' ').filter(Boolean);
  for (const [field, aliases] of Object.entries(aliasMap)) {
    for (const alias of aliases) {
      if (norm === alias) return field;
    }
    // Single short alias token appearing standalone (e.g. "au" inside "au ppm").
    for (const alias of aliases) {
      if (!alias.includes(' ') && alias.length <= 4 && tokens.includes(alias)) return field;
    }
  }
  return null;
}

/** Map a header row (array of cells) to { colIndex: field } + unrecognized list. */
function buildColumnMap(headerCells, aliasMap) {
  const colFieldMap = Object.create(null);
  const unrecognized = [];
  const seen = new Set();
  headerCells.forEach((cell, idx) => {
    const raw = cellToString(cell);
    if (!raw) return;
    const field = matchField(normHeader(raw), aliasMap);
    if (field && !seen.has(field)) {
      colFieldMap[idx] = field;
      seen.add(field);
    } else if (!field) {
      unrecognized.push(raw);
    }
  });
  return { colFieldMap, unrecognized };
}

function countMatches(headerCells, aliasMap) {
  const fields = new Set();
  for (const cell of headerCells) {
    const field = matchField(normHeader(cellToString(cell)), aliasMap);
    if (field) fields.add(field);
  }
  return fields;
}

// ── Cell value coercion ─────────────────────────────────────────────────────

/** Raw cell → primitive (number stays number; formula→result; richText→text). */
function cellToValue(cell) {
  if (cell === null || cell === undefined) return null;
  if (typeof cell === 'number') return cell;
  if (typeof cell === 'boolean') return cell;
  if (cell instanceof Date) return cell.toISOString();
  if (typeof cell === 'object') {
    if (cell.result !== undefined) return cell.result;
    if (cell.text !== undefined) return cell.text;
    if (Array.isArray(cell.richText)) return cell.richText.map(r => r.text || '').join('');
    if (cell.error !== undefined) return null;
    return null;
  }
  return cell;
}

function cellToString(cell) {
  const v = cellToValue(cell);
  if (v === null || v === undefined) return null;
  const s = String(v).trim();
  return s === '' ? null : s;
}

/** Parse a numeric cell. Blank → null. Handles comma decimals. Non-numeric → null. */
function parseNumber(cell) {
  const v = cellToValue(cell);
  if (v === null || v === undefined || v === '') return null;
  if (typeof v === 'number') return Number.isFinite(v) ? v : null;
  let s = String(v).trim();
  if (s === '') return null;
  s = s.replace(/\s+/g, '');
  // Spanish comma decimal: "1.234,56" → "1234.56"; "12,5" → "12.5".
  if (s.includes(',') && !s.includes('.')) s = s.replace(/,/g, '.');
  else s = s.replace(/,/g, '');
  const n = parseFloat(s);
  return Number.isFinite(n) ? n : null;
}

function parseText(cell) {
  return cellToString(cell);
}

// ── File reading ──────────────────────────────────────────────────────────────

function parseCsv(content) {
  const text = content.replace(/^﻿/, '');
  const rows = [];
  let row = [];
  let field = '';
  let inQuotes = false;
  for (let i = 0; i < text.length; i++) {
    const ch = text[i];
    if (inQuotes) {
      if (ch === '"') {
        if (text[i + 1] === '"') { field += '"'; i++; }
        else inQuotes = false;
      } else field += ch;
    } else if (ch === '"') {
      inQuotes = true;
    } else if (ch === ',') {
      row.push(field); field = '';
    } else if (ch === '\n') {
      row.push(field); rows.push(row); row = []; field = '';
    } else if (ch === '\r') {
      // handled by the \n branch; ignore lone CR
    } else {
      field += ch;
    }
  }
  if (field !== '' || row.length > 0) { row.push(field); rows.push(row); }
  return rows.map(r => r.map(c => (c === '' ? null : c)));
}

/** Legacy binary .xls via SheetJS. Optional dep — clear error if not installed. */
async function readRowsXls(filePath) {
  let mod;
  try {
    mod = await import(/* webpackIgnore: true */ /* turbopackIgnore: true */ 'xlsx');
  } catch {
    throw new Error('xls_parser_unavailable');
  }
  const XLSX = mod?.default?.read ? mod.default : mod;
  const wb = XLSX.read(fs.readFileSync(filePath), { type: 'buffer', cellDates: true });
  const ws = wb.Sheets[wb.SheetNames[0]];
  if (!ws) return [];
  // header:1 → array-of-arrays; raw keeps numbers/Dates as primitives (handled
  // downstream by cellToValue); blankrows mirrors exceljs includeEmpty.
  return XLSX.utils.sheet_to_json(ws, { header: 1, raw: true, defval: null, blankrows: true });
}

/** Read a tabular file into an array of rows (each a 0-indexed cell array). */
async function readRows(filePath) {
  const ext = path.extname(filePath).toLowerCase();
  if (ext === '.csv') {
    return parseCsv(fs.readFileSync(filePath, 'utf8'));
  }
  if (ext === '.xls') {
    return readRowsXls(filePath);
  }
  const wb = new ExcelJS.Workbook();
  await wb.xlsx.readFile(filePath); // exceljs reads .xlsx only (zip format)
  const ws = wb.worksheets[0];
  if (!ws) return [];
  const rows = [];
  ws.eachRow({ includeEmpty: true }, (excelRow) => {
    const values = excelRow.values || [];
    const arr = [];
    for (let i = 1; i < values.length; i++) arr.push(values[i] ?? null);
    rows.push(arr);
  });
  return rows;
}

/** Scan the first rows for the header (≥ MIN_HEADER_MATCHES recognised columns). */
function findHeaderRowIndex(rows, aliasMap) {
  const limit = Math.min(MAX_HEADER_SCAN, rows.length);
  let best = { idx: -1, matches: 0 };
  for (let i = 0; i < limit; i++) {
    const matches = countMatches(rows[i] || [], aliasMap).size;
    if (matches > best.matches) best = { idx: i, matches };
  }
  return best.matches >= MIN_HEADER_MATCHES ? best.idx : -1;
}

// ── detectTabularType ───────────────────────────────────────────────────────

/**
 * Classify a tabular file as 'assay' | 'drill' | 'unknown'.
 *
 * @param {string|null} filePath  Path to read when `headers` is not supplied.
 * @param {string[]}    [headers] Explicit header cells (skips file read).
 * @returns {Promise<'assay'|'drill'|'unknown'>}
 */
export async function detectTabularType(filePath, headers) {
  let headerCells = Array.isArray(headers) ? headers : null;
  if (!headerCells) {
    const rows = await readRows(filePath);
    // Pick whichever alias map yields the strongest header row.
    const assayIdx = findHeaderRowIndex(rows, ASSAY_ALIASES);
    const drillIdx = findHeaderRowIndex(rows, DRILL_ALIASES);
    const idx = assayIdx >= 0 ? assayIdx : drillIdx >= 0 ? drillIdx : 0;
    headerCells = rows[idx] || [];
  }

  const assayFields = countMatches(headerCells, ASSAY_ALIASES);
  const drillFields = countMatches(headerCells, DRILL_ALIASES);

  const hasHoleId = drillFields.has('hole_id');
  const hasSampleId = assayFields.has('sample_id');
  const gradeCount = ['au_ppm', 'ag_ppm', 'cu_pct'].filter(f => assayFields.has(f)).length;
  const drillGeomCount = ['azimuth', 'dip', 'total_depth_m'].filter(f => drillFields.has(f)).length;
  const collarCount = ['collar_easting', 'collar_northing', 'collar_elevation'].filter(f => drillFields.has(f)).length;

  // Strong assay signal wins first: a sample id plus ≥2 grade columns is an
  // assay table even when an incidental hole_id/coordinate column also matches
  // (geochem files carry a HoleID link and many element columns).
  if (hasSampleId && gradeCount >= 2) return 'assay';
  if (hasHoleId && (drillGeomCount >= 1 || collarCount >= 1)) return 'drill';
  if (hasSampleId && gradeCount >= 1) return 'assay';
  if (gradeCount >= 2) return 'assay';
  if (drillGeomCount >= 2) return 'drill';
  return 'unknown';
}

// ── loadAssaySamples ──────────────────────────────────────────────────────────

/**
 * Load assay samples from a tabular file into mines_assay_samples.
 *
 * @param {string} filePath
 * @param {string} projectId
 * @param {string} sourceDocumentId
 * @returns {Promise<{ status, rows_loaded, rows_skipped, unrecognized_columns: string[], error? }>}
 */
export async function loadAssaySamples(filePath, projectId, sourceDocumentId) {
  const rows = await readRows(filePath);
  const headerIdx = findHeaderRowIndex(rows, ASSAY_ALIASES);
  if (headerIdx < 0) {
    return {
      status: 'error',
      error: 'no_header',
      rows_loaded: 0,
      rows_skipped: 0,
      unrecognized_columns: [],
    };
  }

  const { colFieldMap, unrecognized } = buildColumnMap(rows[headerIdx], ASSAY_ALIASES);

  // The cu_pct schema column is a PERCENT. If the source column that mapped to
  // it is reported in ppm (e.g. "Cu_ppm"), convert ppm→pct (÷10000) so we never
  // store a ppm magnitude in a pct field (would be 10000× too high).
  let cuFromPpm = false;
  for (const [idx, field] of Object.entries(colFieldMap)) {
    if (field !== 'cu_pct') continue;
    const h = normHeader(cellToString(rows[headerIdx][Number(idx)]) || '');
    const tokens = h.split(' ');
    if (tokens.includes('ppm') && !tokens.includes('pct') && !tokens.includes('pc')) cuFromPpm = true;
  }

  const db = getDb();
  const now = nowIso();

  const insert = db.prepare(`
    INSERT INTO mines_assay_samples
      (id, project_id, drillhole_id, sample_id, easting, northing, elevation,
       sample_type, width_m, au_ppm, ag_ppm, cu_pct, other_grades_json,
       qa_qc_status, source_document_id, confidence, extracted_by_agent_id,
       created_at, updated_at)
    VALUES (?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, 1.0, NULL, ?, ?)
    ON CONFLICT(project_id, source_document_id, sample_id) DO UPDATE SET
      easting = excluded.easting,
      northing = excluded.northing,
      elevation = excluded.elevation,
      sample_type = excluded.sample_type,
      width_m = excluded.width_m,
      au_ppm = excluded.au_ppm,
      ag_ppm = excluded.ag_ppm,
      cu_pct = excluded.cu_pct,
      updated_at = excluded.updated_at
  `);

  let rows_loaded = 0;
  let rows_skipped = 0;

  db.prepare('BEGIN').run();
  try {
    for (let r = headerIdx + 1; r < rows.length; r++) {
      const cells = rows[r] || [];
      const f = extractFields(cells, colFieldMap);
      if (isBlankRow(f)) continue; // fully empty trailing row — not counted

      const sampleId = parseText(f.sample_id);
      if (!sampleId) { rows_skipped++; continue; }

      const cuRaw = parseNumber(f.cu_pct);
      const cu = cuFromPpm && cuRaw != null ? cuRaw / 10000 : cuRaw;

      insert.run(
        newId('mas'),
        projectId,
        sampleId,
        parseNumber(f.easting),
        parseNumber(f.northing),
        parseNumber(f.elevation),
        parseText(f.sample_type),
        parseNumber(f.width_m),
        parseNumber(f.au_ppm),
        parseNumber(f.ag_ppm),
        cu,
        sourceDocumentId || null,
        now,
        now
      );
      rows_loaded++;
    }
    db.prepare('COMMIT').run();
  } catch (err) {
    try { db.prepare('ROLLBACK').run(); } catch {}
    throw err;
  }

  return { status: 'ok', rows_loaded, rows_skipped, unrecognized_columns: unrecognized };
}

// ── loadDrillHoles ────────────────────────────────────────────────────────────

/**
 * Load drill holes from a tabular file into mines_drill_holes.
 * Idempotent on (project_id, hole_id) — duplicates count as skipped.
 *
 * @returns {Promise<{ status, rows_loaded, rows_skipped, unrecognized_columns: string[], error? }>}
 */
export async function loadDrillHoles(filePath, projectId, sourceDocumentId) {
  const rows = await readRows(filePath);
  const headerIdx = findHeaderRowIndex(rows, DRILL_ALIASES);
  if (headerIdx < 0) {
    return {
      status: 'error',
      error: 'no_header',
      rows_loaded: 0,
      rows_skipped: 0,
      unrecognized_columns: [],
    };
  }

  const { colFieldMap, unrecognized } = buildColumnMap(rows[headerIdx], DRILL_ALIASES);
  const db = getDb();
  const now = nowIso();

  const insert = db.prepare(`
    INSERT OR IGNORE INTO mines_drill_holes
      (id, project_id, hole_id, collar_easting, collar_northing, collar_elevation,
       azimuth, dip, total_depth_m, crs, qa_qc_status, source_document_id,
       confidence, extracted_by_agent_id, created_at, updated_at)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, 1.0, NULL, ?, ?)
  `);

  let rows_loaded = 0;
  let rows_skipped = 0;

  db.prepare('BEGIN').run();
  try {
    for (let r = headerIdx + 1; r < rows.length; r++) {
      const cells = rows[r] || [];
      const f = extractFields(cells, colFieldMap);
      if (isBlankRow(f)) continue;

      const holeId = parseText(f.hole_id);
      if (!holeId) { rows_skipped++; continue; }

      const res = insert.run(
        newId('mdh'),
        projectId,
        holeId,
        parseNumber(f.collar_easting),
        parseNumber(f.collar_northing),
        parseNumber(f.collar_elevation),
        parseNumber(f.azimuth),
        parseNumber(f.dip),
        parseNumber(f.total_depth_m),
        sourceDocumentId || null,
        now,
        now
      );
      if (res.changes > 0) rows_loaded++;
      else rows_skipped++; // duplicate (project_id, hole_id)
    }
    db.prepare('COMMIT').run();
  } catch (err) {
    try { db.prepare('ROLLBACK').run(); } catch {}
    throw err;
  }

  return { status: 'ok', rows_loaded, rows_skipped, unrecognized_columns: unrecognized };
}

// ── helpers ───────────────────────────────────────────────────────────────────

function extractFields(cells, colFieldMap) {
  const out = Object.create(null);
  for (const [idx, field] of Object.entries(colFieldMap)) {
    out[field] = cells[Number(idx)] ?? null;
  }
  return out;
}

function isBlankRow(fields) {
  return Object.values(fields).every(v => {
    const s = cellToString(v);
    return s === null;
  });
}
