'use strict';

let telemetry;
try { telemetry = require('../telemetry'); } catch { telemetry = { trackError() {} }; }

/**
 * Backfill loop: upgrades old memories with domain/topic classification
 * and vector embeddings. Runs in the background.
 *
 * - Processes ALL memories with content (regardless of extraction_status)
 *   that lack domain/topic classification or embeddings
 * - Classification is rule-based (free, no API calls) — runs in large batches
 * - Embedding uses the auto-detected provider — batches up to provider max
 * - Tracks progress via kv_store so it resumes across restarts
 * - Automatically stops when all memories are processed
 */

const brain = require('../brain');
const { stripNoise } = require('../extraction/noise-stripper');
const { extractHeuristics } = require('../extraction/heuristic-extractor');
const { linkKnowledgeToEntities } = require('../extraction/entity-normalizer');
const { generateMemoryIndex } = require('../extraction/indexer');
let embeddings;
try { embeddings = require('../embeddings'); } catch { embeddings = null; }

const CLASSIFY_BATCH = 500;   // Classification is free (no API calls), larger batches are fine
const EMBED_BATCH = 200;      // Embedding batches — Ollama handles 32 at a time internally, Google/OpenAI handle 100+
const HEURISTIC_BATCH = 500;
const ENTITY_BATCH = 500;
const INDEX_BATCH = 500;
const KV_CLASSIFY_CURSOR = 'backfill:classify:cursor';
const KV_EMBED_CURSOR = 'backfill:embed:cursor';
const KV_HEURISTIC_CURSOR = 'backfill:heuristic:cursor';
const KV_ENTITY_CURSOR = 'backfill:entity:cursor';
const KV_INDEX_CURSOR = 'backfill:index:cursor';
const KV_NOISE_CURSOR = 'backfill:noise:cursor';
const KV_BACKFILL_DONE = 'backfill:done';

const VALID_DOMAINS = ['work', 'personal', 'social', 'technical', 'admin'];

// ── Cheap "work remaining" signals ──────────────────────────────────────────
// This loop runs every 30s ON THE MAIN THREAD, and better-sqlite3 is synchronous,
// so any unbounded query here blocks Wall-E's entire event loop — HTTP included —
// for the full duration of the scan. On a multi-GB brain that means the daemon
// stops answering /api/* (e.g. the setup page's provider list) for tens of seconds
// at a time. So every per-tick "is there work left?" check MUST stay bounded — no
// full-table content scans.
//
// `length(content) > 10` forces reading each candidate row's (often overflow-paged)
// content, so counting embeddable memories is the one unavoidable full scan. It
// only moves as memories are ingested, so cache it briefly instead of paying it
// every tick.
let _embeddableCache = { value: null, at: 0 };
const EMBEDDABLE_TTL_MS = 10 * 60 * 1000;
function embeddableMemoryTotal() {
  const now = Date.now();
  if (_embeddableCache.value != null && now - _embeddableCache.at < EMBEDDABLE_TTL_MS) {
    return _embeddableCache.value;
  }
  const c = brain.getDb().prepare(
    "SELECT count(*) AS c FROM memories WHERE content IS NOT NULL AND length(content) > 10"
  ).get().c;
  _embeddableCache = { value: c, at: now };
  return c;
}

/**
 * Run one backfill iteration. Call periodically from the daemon.
 * Returns { classified, embedded, done } counts.
 */
async function runOnce(opts = {}) {
  // Skip if already fully backfilled
  if (brain.getKv(KV_BACKFILL_DONE) === '1') {
    return { classified: 0, embedded: 0, heuristic: 0, entityLinked: 0, indexed: 0, noiseStripped: 0, done: true };
  }

  const result = { classified: 0, embedded: 0, heuristic: 0, entityLinked: 0, indexed: 0, noiseStripped: 0, done: false };

  // Phase 0: Noise stripping for memories ingested before the stripper was added
  try {
    result.noiseStripped = await noiseStripBatch();
  } catch (err) {
    console.error('[backfill] Noise stripping error:', err.message);
  }

  // Phase 1: Classify unclassified memories (free, rule-based).
  // Cap at MAX_CLASSIFY_PER_TICK to keep each tick under ~10s and avoid crashing
  // on large backlogs (191K+ memories). Multiple ticks will catch up.
  const MAX_CLASSIFY_PER_TICK = 5000;
  try {
    let batch;
    do {
      batch = await classifyBatch(opts);
      result.classified += batch;
      if (batch >= CLASSIFY_BATCH && result.classified % 5000 < CLASSIFY_BATCH) {
        console.log(`[backfill] Classified ${result.classified} memories this tick...`);
      }
      if (batch >= CLASSIFY_BATCH) await new Promise(r => setImmediate(r));
    } while (batch >= CLASSIFY_BATCH && result.classified < MAX_CLASSIFY_PER_TICK);
  } catch (err) {
    console.error('[backfill] Classification error:', err.message);
    telemetry.trackError('backfill-classify', err);
  }

  // Phase 2: Embed memories missing embeddings — run multiple batches per tick
  // Quick Ollama readiness check — after laptop sleep, Ollama needs a few seconds
  // to reload. Skip this tick rather than wasting it on failed/slow batches.
  // Gate on the cheap remaining-count BEFORE the readiness probe: when embedding is
  // already complete, embedBatch's anti-join would otherwise SCAN the whole memories
  // table just to return 0 every tick. countUnembedded() is bounded (indexed counts).
  if (embeddings && embeddings.isAvailable() && countUnembedded() > 0 && await _isEmbeddingProviderReady()) {
    const MAX_BATCHES_PER_TICK = 5; // Up to 5 x EMBED_BATCH per tick for faster backfill
    for (let b = 0; b < MAX_BATCHES_PER_TICK; b++) {
      try {
        const embedded = await embedBatch();
        result.embedded += embedded;
        if (embedded < EMBED_BATCH) break; // Short batch = near end, stop early
        // Yield to event loop between batches
        await new Promise(r => setImmediate(r));
      } catch (err) {
        console.error('[backfill] Embedding error:', err.message);
      telemetry.trackError('backfill-embed', err);
        break;
      }
    }
    if (result.embedded > 0 && result.embedded % 1000 < EMBED_BATCH * MAX_BATCHES_PER_TICK) {
      console.log(`[backfill] Embedded ${result.embedded} memories this tick`);
    }
  }

  // Phase 3: Heuristic knowledge extraction (free, no API calls)
  try {
    result.heuristic = await heuristicBatch();
  } catch (err) {
    console.error('[backfill] Heuristic extraction error:', err.message);
  }

  // Phase 4: Entity linking for unlinked knowledge (free, no API calls)
  try {
    result.entityLinked = await entityLinkBatch();
  } catch (err) {
    console.error('[backfill] Entity linking error:', err.message);
  }

  // Phase 5: Memory index generation (free, no API calls)
  try {
    result.indexed = await indexBatch();
  } catch (err) {
    console.error('[backfill] Indexing error:', err.message);
  }

  // Check if we're done
  const unclassified = countUnclassified();
  const unembedded = embeddings && embeddings.isAvailable() ? countUnembedded() : 0;

  if (unembedded === 0 && unclassified <= 20) {
    // Mark done when all memories are embedded. A small number of unclassified
    // memories (tool-call artifacts, etc.) is acceptable — they're still embedded.
    brain.setKv(KV_BACKFILL_DONE, '1');
    result.done = true;
    console.log('[backfill] All memories backfilled');
  } else if (result.classified === 0 && result.embedded === 0 && unembedded > 0) {
    // Cursor reached end but some memories still unembedded (rare partial failures).
    // Reset embed cursor to retry from the beginning.
    brain.setKv(KV_EMBED_CURSOR, '0');
  }

  return result;
}

/**
 * Classify a batch of memories that have no domain yet.
 * Classifies ALL memories (not just extraction_status='done') — classification
 * is free (rule-based, no API calls) and benefits semantic search.
 */
async function classifyBatch(opts = {}) {
  const db = brain.getDb();

  // Self-draining: select straight from the unclassified set (domain IS NULL) with a
  // plain LIMIT — no `id > cursor` / `ORDER BY id ASC`. The old cursor+ORDER BY form
  // made SQLite drive off idx_memories_domain and build a TEMP B-TREE over EVERY
  // unclassified row (reading each one's content via a table seek) before applying
  // LIMIT — so a single call read 80k+ rows' content and blocked the event loop for
  // tens of seconds. classifyMemory always assigns a non-null domain, so classified
  // rows leave the set and the next LIMIT picks up where this one left off. This is
  // bounded to CLASSIFY_BATCH content reads per call and can't strand rows.
  const batch = db.prepare(`
    SELECT id, content, source, source_channel FROM memories
    WHERE domain IS NULL AND content IS NOT NULL AND length(content) > 10
    LIMIT ?
  `).all(CLASSIFY_BATCH);

  if (batch.length === 0) return 0;

  // Classify in a single transaction for speed
  const classify = db.transaction(() => {
    for (const mem of batch) {
      const { domain, topic } = classifyMemory(mem);
      brain.updateMemoryClassification(mem.id, domain, topic);
    }
  });
  classify();

  return batch.length;
}

/**
 * Rule-based memory classification. No API call needed.
 * Uses source, channel, and keyword patterns to determine domain/topic.
 */
function classifyMemory(mem) {
  const content = (mem.content || '').toLowerCase();
  const source = mem.source || '';
  const channel = (mem.source_channel || '').toLowerCase();

  let domain = 'work'; // default
  let topic = null;

  // Domain classification from channel context
  if (channel.includes('random') || channel.includes('fun') || channel.includes('social')) {
    domain = 'social';
  } else if (channel.includes('personal') || channel.includes('health') || channel.includes('fitness')) {
    domain = 'personal';
  } else if (channel.includes('ops') || channel.includes('infra') || channel.includes('devops') || channel.includes('sre') || channel.includes('engineering') || channel.startsWith('eng-')) {
    domain = 'technical';
  } else if (channel.includes('admin') || channel.includes('hr') || channel.includes('finance') || channel.includes('legal')) {
    domain = 'admin';
  }

  // Override with content-based patterns
  const techPatterns = /\b(deploy|ci|pipeline|docker|kubernetes|terraform|git|commit|merge|branch|bug|error|crash|api|endpoint|database|sql|redis|cache|server|node|python|react|test|lint|postgres|mysql|sqlite|migration|build|release|infra|monitor|alert|incident|outage|auth|security)\b/;
  const personalPatterns = /\b(family|vacation|travel|health|gym|workout|hobby|birthday|anniversary|weekend plan|day off)\b/;
  const adminPatterns = /\b(meeting|calendar|schedule|reminder|todo|task|deadline|review|standup|1:1|one-on-one|okr|goal|budget)\b/;
  const socialPatterns = /\b(lunch|coffee|happy hour|congrats|welcome|goodbye|farewell|thank you|great job|shoutout)\b/;

  if (techPatterns.test(content)) domain = 'technical';
  else if (personalPatterns.test(content)) domain = 'personal';
  else if (socialPatterns.test(content)) domain = 'social';
  else if (adminPatterns.test(content)) domain = 'admin';

  // Topic extraction: find the most specific keyword
  const topicPatterns = [
    { pattern: /\b(deploy|deployment|ship|release|rollback)\b/, topic: 'deployment' },
    { pattern: /\b(ci|pipeline|build|github.actions|jenkins)\b/, topic: 'ci-pipeline' },
    { pattern: /\b(auth|authentication|login|oauth|sso|jwt)\b/, topic: 'auth' },
    { pattern: /\b(database|sql|postgres|mysql|sqlite|migration)\b/, topic: 'database' },
    { pattern: /\b(hiring|interview|candidate|recruit|onboard)\b/, topic: 'hiring' },
    { pattern: /\b(review|feedback|perf|performance.review|promotion)\b/, topic: 'performance' },
    { pattern: /\b(incident|outage|page|alert|oncall|on-call)\b/, topic: 'incidents' },
    { pattern: /\b(docker|container|kubernetes|k8s|helm)\b/, topic: 'infrastructure' },
    { pattern: /\b(test|testing|unit.test|e2e|coverage)\b/, topic: 'testing' },
    { pattern: /\b(api|endpoint|rest|graphql|grpc)\b/, topic: 'api' },
    { pattern: /\b(slack|channel|message|thread)\b/, topic: 'slack' },
    { pattern: /\b(wall-?e|walle|digital.twin|brain)\b/, topic: 'wall-e' },
    { pattern: /\b(budget|cost|billing|expense|invoice)\b/, topic: 'finance' },
    { pattern: /\b(meeting|standup|sync|1:1|retro)\b/, topic: 'meetings' },
    { pattern: /\b(design|ui|ux|figma|mockup|prototype)\b/, topic: 'design' },
    { pattern: /\b(security|vuln|cve|patch|encrypt)\b/, topic: 'security' },
    { pattern: /\b(monitor|metric|dashboard|grafana|datadog)\b/, topic: 'observability' },
  ];

  for (const { pattern, topic: t } of topicPatterns) {
    if (pattern.test(content)) {
      topic = t;
      break;
    }
  }

  // Fallback topic from channel name
  if (!topic && channel) {
    topic = channel.replace(/^#/, '').slice(0, 50);
  }

  return { domain, topic };
}

/**
 * Embed a batch of memories that have no embedding yet.
 */
async function embedBatch() {
  if (!embeddings || !embeddings.isAvailable()) return 0;

  const db = brain.getDb();
  const activeModel = embeddings.getEmbeddingModel();
  if (!activeModel) return 0;

  const cursor = brain.getKv(KV_EMBED_CURSOR) || '0';

  // Find memories not yet embedded with the ACTIVE model.
  // Embed ALL memories (not just extraction_status='done') — pending/failed memories
  // still have content that's valuable for semantic search.
  const batch = db.prepare(`
    SELECT m.id, m.content FROM memories m
    LEFT JOIN embedding_map em ON em.entity_id = m.id AND em.model = ?
    WHERE em.rowid IS NULL AND m.content IS NOT NULL AND length(m.content) > 10 AND m.id > ?
    ORDER BY m.id ASC LIMIT ?
  `).all(activeModel, cursor, EMBED_BATCH);

  if (batch.length === 0) return 0;

  const texts = batch.map(m => (m.content || '').substring(0, 512));
  const vectors = await embeddings.batchComputeEmbeddings(texts);
  if (!vectors) return 0;

  let stored = 0;
  for (let i = 0; i < batch.length; i++) {
    if (vectors[i]) {
      embeddings.storeEmbedding(batch[i].id, 'memory', vectors[i]);
      stored++;
    }
  }

  // Always advance cursor — the LEFT JOIN will re-find any skipped memories
  // on subsequent runs once the cursor wraps or backfill resets. In practice,
  // partial vector responses are extremely rare (API returns all or nothing).
  brain.setKv(KV_EMBED_CURSOR, batch[batch.length - 1].id);
  return stored;
}

async function heuristicBatch() {
  const db = brain.getDb();
  const cursor = brain.getKv(KV_HEURISTIC_CURSOR) || '0';
  const ownerName = brain.getOwnerName ? brain.getOwnerName() : 'owner';
  const batch = db.prepare(`
    SELECT id, content, source, timestamp FROM memories
    WHERE content IS NOT NULL AND length(content) > 20 AND id > ?
    ORDER BY id ASC LIMIT ?
  `).all(cursor, HEURISTIC_BATCH);
  if (batch.length === 0) return 0;
  let extracted = 0;
  const insert = db.transaction(() => {
    for (const mem of batch) {
      const entries = extractHeuristics(mem, ownerName);
      for (const entry of entries) {
        try {
          brain.insertKnowledge({ ...entry, source_memory_ids: JSON.stringify(entry.source_memory_ids) });
          extracted++;
        } catch {}
      }
    }
  });
  insert();
  brain.setKv(KV_HEURISTIC_CURSOR, batch[batch.length - 1].id);
  return extracted;
}

async function entityLinkBatch() {
  const db = brain.getDb();
  const cursor = brain.getKv(KV_ENTITY_CURSOR) || '0';
  const ownerName = brain.getOwnerName ? brain.getOwnerName() : 'owner';
  const batch = db.prepare(`
    SELECT id, subject, object FROM knowledge
    WHERE subject_entity_id IS NULL AND id > ?
    ORDER BY id ASC LIMIT ?
  `).all(cursor, ENTITY_BATCH);
  if (batch.length === 0) return 0;
  let linked = 0;
  const link = db.transaction(() => {
    for (const k of batch) {
      try {
        const result = linkKnowledgeToEntities(k, ownerName);
        if (result.subjectEntity || result.objectEntity) linked++;
      } catch {}
    }
  });
  link();
  brain.setKv(KV_ENTITY_CURSOR, batch[batch.length - 1].id);
  return linked;
}

async function indexBatch() {
  const db = brain.getDb();
  const cursor = brain.getKv(KV_INDEX_CURSOR) || '0';
  const batch = db.prepare(`
    SELECT m.id, m.content, m.domain, m.topic, m.timestamp FROM memories m
    LEFT JOIN memory_index mi ON mi.memory_id = m.id
    WHERE mi.id IS NULL AND m.content IS NOT NULL AND length(m.content) > 20 AND m.id > ?
    ORDER BY m.id ASC LIMIT ?
  `).all(cursor, INDEX_BATCH);
  if (batch.length === 0) return 0;
  let count = 0;
  const insert = db.transaction(() => {
    for (const mem of batch) {
      const entry = generateMemoryIndex(mem);
      if (entry) {
        try { brain.insertMemoryIndex(entry); count++; } catch {}
      }
    }
  });
  insert();
  brain.setKv(KV_INDEX_CURSOR, batch[batch.length - 1].id);
  return count;
}

const NOISE_BATCH = 500;
const KV_NOISE_DONE = 'backfill:noise:done';

async function noiseStripBatch() {
  // One-time legacy sweep: new content is stripped at ingest, so once we've swept the
  // whole table there's nothing left to do — skip entirely instead of re-scanning.
  if (brain.getKv(KV_NOISE_DONE) === '1') return 0;

  const db = brain.getDb();
  const cursor = brain.getKv(KV_NOISE_CURSOR) || '0';
  // Scan a bounded id-window (LIMIT applies to SCANNED rows) and match the legacy
  // <system-reminder> tag in JS. The old query put `content LIKE '%…%'` in SQL, so with
  // only a handful of tagged rows it never filled the LIMIT and the cursor — advanced to
  // the last MATCHED id — barely moved, making every 30s tick re-scan the rest of the
  // table reading content. Advancing past the last SCANNED row guarantees forward
  // progress and lets the sweep terminate.
  const batch = db.prepare(
    'SELECT id, content FROM memories WHERE id > ? ORDER BY id ASC LIMIT ?'
  ).all(cursor, NOISE_BATCH);
  if (batch.length === 0) { brain.setKv(KV_NOISE_DONE, '1'); return 0; }

  let stripped = 0;
  const update = db.transaction(() => {
    for (const mem of batch) {
      const content = mem.content;
      if (!content || content.length <= 50 || !content.includes('<system-reminder>')) continue;
      const cleaned = stripNoise(content);
      if (cleaned !== content) {
        db.prepare('UPDATE memories SET content_raw = content, content = ? WHERE id = ?').run(cleaned, mem.id);
        stripped++;
      }
    }
  });
  update();
  brain.setKv(KV_NOISE_CURSOR, batch[batch.length - 1].id);
  // Short batch ⇒ reached the end of the table ⇒ sweep complete.
  if (batch.length < NOISE_BATCH) brain.setKv(KV_NOISE_DONE, '1');
  return stripped;
}

// Capped at 21: the only consumers are the done-gate (`<= 20`) and a diagnostic log
// line. The exact count would read content for every one of 80k+ unclassified rows
// each tick (full scan); the LIMIT lets SQLite stop after the first 21 matches.
function countUnclassified() {
  try {
    return brain.getDb().prepare(
      "SELECT count(*) AS c FROM (SELECT 1 FROM memories WHERE domain IS NULL AND content IS NOT NULL AND length(content) > 10 LIMIT 21)"
    ).get().c;
  } catch { return 0; }
}

// Cheap: the accurate anti-join (memories LEFT JOIN embedding_map ... rowid IS NULL)
// is a full SCAN of memories reading content — tens of seconds cold, every tick.
// Instead derive remaining work from two bounded counts: the (cached) embeddable
// total minus the indexed count of memory-type embeddings for the active model.
// Stale embeddings can make this slightly under-count, which is acceptable: it only
// gates "is embedding done?" and mirrors the existing `unclassified <= 20` tolerance
// — a handful of unembedded rows won't meaningfully hurt semantic search.
function countUnembedded() {
  try {
    const activeModel = embeddings && embeddings.getEmbeddingModel();
    if (!activeModel) return 0;
    const embedded = brain.getDb().prepare(
      "SELECT count(*) AS c FROM embedding_map WHERE model = ? AND entity_type = 'memory'"
    ).get(activeModel).c;
    return Math.max(0, embeddableMemoryTotal() - embedded);
  } catch { return 0; }
}

/**
 * Reset backfill state so it re-processes everything.
 */
function reset() {
  brain.setKv(KV_BACKFILL_DONE, null);
  brain.setKv(KV_CLASSIFY_CURSOR, null);
  brain.setKv(KV_EMBED_CURSOR, null);
  brain.setKv(KV_HEURISTIC_CURSOR, null);
  brain.setKv(KV_ENTITY_CURSOR, null);
  brain.setKv(KV_INDEX_CURSOR, null);
  brain.setKv(KV_NOISE_CURSOR, null);
  brain.setKv(KV_NOISE_DONE, null);
}

/**
 * Get backfill progress stats.
 */
function getProgress() {
  const unclassified = countUnclassified();
  const unembedded = embeddings && embeddings.isAvailable() ? countUnembedded() : 0;
  const done = brain.getKv(KV_BACKFILL_DONE) === '1';
  return { unclassified, unembedded, done };
}

/**
 * Startup integrity check. Call once on daemon boot.
 * Detects if backfill was marked "done" but a large proportion of memories
 * are actually unembedded (e.g., the old extraction_status='done' filter
 * skipped most memories). Auto-resets if coverage is below 80%.
 */
function checkIntegrity() {
  try {
    const done = brain.getKv(KV_BACKFILL_DONE) === '1';
    if (!done) return; // not marked done, nothing to check

    if (!embeddings || !embeddings.isAvailable()) return;

    const db = brain.getDb();
    const totalEmbeddable = db.prepare(
      "SELECT count(*) as c FROM memories WHERE content IS NOT NULL AND length(content) > 10"
    ).get().c;
    if (totalEmbeddable === 0) return;

    const embedded = db.prepare(
      "SELECT count(*) as c FROM embedding_map WHERE model = ?"
    ).get(embeddings.getEmbeddingModel()).c;

    const coverage = embedded / totalEmbeddable;
    if (coverage < 0.8) {
      console.log(`[backfill] Integrity check: only ${Math.round(coverage * 100)}% embedded (${embedded.toLocaleString()} / ${totalEmbeddable.toLocaleString()}). Auto-resetting.`);
      reset();
    }
  } catch (err) {
    console.error('[backfill] Integrity check error:', err.message);
  }
}

/**
 * Quick readiness check for the embedding provider.
 * After laptop sleep/wake, Ollama may need seconds to reload the model.
 * A fast probe avoids wasting a full tick on a failed batch.
 */
async function _isEmbeddingProviderReady() {
  if (!embeddings) return false;
  try {
    // Try embedding a single short text with a tight timeout
    const result = await Promise.race([
      embeddings.batchComputeEmbeddings(['readiness probe']),
      new Promise((_, reject) => setTimeout(() => reject(new Error('probe timeout')), 5000)),
    ]);
    return result && result.length > 0;
  } catch {
    console.log('[backfill] Embedding provider not ready (post-sleep?) — skipping this tick');
    return false;
  }
}

module.exports = { runOnce, reset, getProgress, checkIntegrity, classifyMemory, VALID_DOMAINS };
