#!/usr/bin/env node
'use strict';
/**
 * Standalone Slack backfill script — NO Claude API tokens used.
 * Directly calls Slack MCP + writes to brain DB.
 * Paginates through all results (up to 20 pages per query).
 *
 * Usage:
 *   node scripts/slack-backfill.js              # full backfill (2022-now)
 *   node scripts/slack-backfill.js incremental  # only new messages
 *   node scripts/slack-backfill.js 2024-01      # single month
 */

const brain = require('../brain');
const slackMcp = require('../tools/slack-mcp');

const PAUSE_MS = 1500;
const MAX_PAGES = 20;
const OWNER_ID = process.env.SLACK_OWNER_USER_ID;
if (!OWNER_ID) throw new Error('SLACK_OWNER_USER_ID env var required');
const SLACK_OWNER_HANDLE = process.env.SLACK_OWNER_HANDLE || 'owner';

async function main() {
  const mode = process.argv[2] || 'full';

  // Init
  brain.initDb();
  const db = brain.getDb();

  if (!(await slackMcp.isAuthenticated())) {
    console.error('Not authenticated with Slack. Run: node tools/slack-mcp.js auth');
    process.exit(1);
  }

  const totalBefore = db.prepare("SELECT count(*) as c FROM memories WHERE source = 'slack'").get().c;
  console.log(`Starting. Mode: ${mode}. Currently have ${totalBefore} Slack messages.\n`);

  let stats = { new_messages: 0, skipped: 0 };

  if (mode === 'incremental') {
    const latest = db.prepare("SELECT max(timestamp) as ts FROM memories WHERE source = 'slack'").get();
    const latestDate = (latest?.ts || '2020-01-01').slice(0, 10);
    console.log(`Incremental from ${latestDate}\n`);
    await pullMonth(db, stats, `from:${SLACK_OWNER_HANDLE} after:${latestDate}`, 'incremental');
  } else if (mode.match(/^\d{4}-\d{2}$/)) {
    // Single month
    const [y, m] = mode.split('-').map(Number);
    const after = `${y}-${String(m).padStart(2, '0')}-01`;
    const before = m === 12 ? `${y + 1}-01-01` : `${y}-${String(m + 1).padStart(2, '0')}-01`;
    console.log(`Single month: ${mode}\n`);
    await pullMonth(db, stats, `from:${SLACK_OWNER_HANDLE} after:${after} before:${before}`, mode);
  } else {
    // Full backfill with checkpoint support
    const months = generateMonths(2022, 6);
    const checkpoint = process.env.WALL_E_CHECKPOINT || '';
    let startIdx = 0;
    if (checkpoint) {
      const idx = months.indexOf(checkpoint);
      if (idx >= 0) {
        startIdx = idx;
        console.log(`Resuming from checkpoint: ${checkpoint}\n`);
      }
    }
    console.log(`Full backfill: ${months.length} months (starting from ${months[startIdx]})\n`);

    for (let mi = startIdx; mi < months.length; mi++) {
      const month = months[mi];
      const existing = db.prepare(
        "SELECT count(*) as c FROM memories WHERE source='slack' AND substr(timestamp,1,7)=?"
      ).get(month).c;

      const [y, m] = month.split('-').map(Number);
      const after = `${y}-${String(m).padStart(2, '0')}-01`;
      const before = m === 12 ? `${y + 1}-01-01` : `${y}-${String(m + 1).padStart(2, '0')}-01`;

      process.stdout.write(`[${month}] ${existing} existing... `);
      const beforeNew = stats.new_messages;
      await pullMonth(db, stats, `from:${SLACK_OWNER_HANDLE} after:${after} before:${before}`, month);
      const added = stats.new_messages - beforeNew;

      const nowCount = db.prepare(
        "SELECT count(*) as c FROM memories WHERE source='slack' AND substr(timestamp,1,7)=?"
      ).get(month).c;
      console.log(`+${added} new → ${nowCount} total`);

      // Emit checkpoint so task runner can save progress
      console.log(`CHECKPOINT:${month}`);
    }
  }

  const totalAfter = db.prepare("SELECT count(*) as c FROM memories WHERE source = 'slack'").get().c;
  console.log(`\nDone! New: ${stats.new_messages}, Skipped dupes: ${stats.skipped}`);
  console.log(`Total: ${totalBefore} → ${totalAfter} (+${totalAfter - totalBefore})`);
}

/**
 * Pull all pages for a single query, paginating with cursors.
 */
async function pullMonth(db, stats, query, label) {
  let cursor = undefined;

  for (let page = 1; page <= MAX_PAGES; page++) {
    try {
      const args = {
        query,
        sort: 'timestamp',
        sort_dir: 'asc',
        limit: 20,
        include_context: false,
        response_format: 'detailed',
      };
      if (cursor) args.cursor = cursor;

      const result = await slackMcp.callSlackMcp('slack_search_public_and_private', args);
      const texts = (result?.content || []).filter(c => c.type === 'text').map(c => c.text);
      let combined = texts.join('\n');
      if (!combined || combined.length < 10) break;

      // The Slack MCP wraps the markdown in a JSON envelope: {"results":"...","pagination_info":"..."}
      let paginationInfo = '';
      try {
        const parsed = JSON.parse(combined);
        combined = parsed.results || combined;
        paginationInfo = parsed.pagination_info || '';
      } catch {}

      // Parse the markdown response
      const messages = parseSlackResults(combined);
      if (messages.length === 0) break;

      for (const msg of messages) {
        const sourceId = `slack-${msg.message_ts}`;
        const exists = db.prepare("SELECT 1 FROM memories WHERE source = 'slack' AND source_id = ?").get(sourceId);
        if (exists) { stats.skipped++; continue; }

        brain.insertMemory({
          source: 'slack',
          source_id: sourceId,
          memory_type: msg.isOwner ? 'message_sent' : 'message_received',
          direction: msg.isOwner ? 'outbound' : 'inbound',
          content: msg.text,
          source_channel: msg.channel,
          participants: msg.participants || msg.sender,
          timestamp: msg.timestamp,
          metadata: JSON.stringify({ channel: msg.channel, sender: msg.sender, channel_id: msg.channelId }),
        });
        stats.new_messages++;
      }

      // Check for next page cursor (in pagination_info or in the text body)
      const cursorMatch = (paginationInfo + '\n' + combined).match(/use cursor `([^`]+)`/);
      if (!cursorMatch) break;
      cursor = cursorMatch[1];

      await sleep(PAUSE_MS);
    } catch (err) {
      if (err.message.includes('401') || err.message.includes('expired')) {
        console.error('\nSlack token expired! Run: node tools/slack-mcp.js auth');
        process.exit(1);
      }
      // Other errors: stop this month, continue
      process.stderr.write(`[${label} p${page}] Error: ${err.message.slice(0, 100)}\n`);
      break;
    }
  }
}

/**
 * Parse the Slack MCP detailed response format.
 * Each result looks like:
 *   ### Result N of M
 *   Channel: #channel-name (ID: CXXX) or DM (ID: DXXX)
 *   Participants: Name1 (ID: UXXX), Name2 (ID: UXXX)
 *   From: Name (ID: UXXX)
 *   Time: 2024-09-04 17:12:28 BST
 *   Message_ts: 1725466348.869379
 *   Text:
 *   The actual message content
 */
function parseSlackResults(text) {
  const messages = [];
  // Split by result headers
  const blocks = text.split(/### Result \d+ of \d+/).slice(1);

  for (const block of blocks) {
    const lines = block.trim().split('\n');
    let channel = 'unknown', channelId = '', sender = '', participants = '', timestamp = '', message_ts = '', content = '';
    let inText = false;

    for (const line of lines) {
      if (inText) {
        if (line.startsWith('---')) break;
        content += (content ? '\n' : '') + line;
        continue;
      }

      const channelMatch = line.match(/^Channel:\s*(?:#?(\S+)|DM|Group DM)\s*\(ID:\s*(\w+)\)/);
      if (channelMatch) {
        channel = channelMatch[1] || (line.includes('DM') ? 'DM' : 'unknown');
        channelId = channelMatch[2] || '';
        continue;
      }

      const participantsMatch = line.match(/^Participants:\s*(.+)/);
      if (participantsMatch) {
        participants = participantsMatch[1].replace(/\s*\(ID:\s*\w+\)/g, '').trim();
        continue;
      }

      const fromMatch = line.match(/^From:\s*(.+?)\s*\(ID:\s*(\w+)\)/);
      if (fromMatch) {
        sender = fromMatch[1].trim() || fromMatch[2];
        continue;
      }

      const timeMatch = line.match(/^Time:\s*(.+)/);
      if (timeMatch) {
        try {
          const d = new Date(timeMatch[1].trim());
          if (!isNaN(d.getTime())) timestamp = d.toISOString();
        } catch {}
        continue;
      }

      const tsMatch = line.match(/^Message_ts:\s*(\S+)/);
      if (tsMatch) {
        message_ts = tsMatch[1];
        // Derive timestamp from message_ts if not already set
        if (!timestamp) {
          try {
            const epoch = parseFloat(message_ts);
            if (epoch > 0) timestamp = new Date(epoch * 1000).toISOString();
          } catch {}
        }
        continue;
      }

      if (line.match(/^Text:\s*$/)) {
        inText = true;
        continue;
      }
      if (line.startsWith('Text:')) {
        inText = true;
        content = line.replace(/^Text:\s*/, '');
        continue;
      }
    }

    content = content.trim();
    if (!content || !message_ts) continue;

    const isOwner = sender.toLowerCase().includes(SLACK_OWNER_HANDLE.split('.')[0]) || sender === OWNER_ID;

    // For DMs/Group DMs, use participant names as channel name instead of raw ID
    if ((channel === 'DM' || channel === 'unknown' || channel === 'Group') && participants) {
      const ownerFirst = (process.env.WALLE_OWNER_NAME || '').split(' ')[0].toLowerCase();
      const others = participants.split(',')
        .map(p => p.trim())
        .filter(p => p && (!ownerFirst || !p.toLowerCase().startsWith(ownerFirst)))
        .map(p => p.split(' ')[0]) // first names only
        .slice(0, 3);
      if (others.length > 0) {
        channel = `DM with ${others.join(', ')}`;
      }
    }

    messages.push({
      channel,
      channelId,
      sender,
      participants,
      timestamp,
      message_ts,
      text: content,
      isOwner,
    });
  }

  return messages;
}

function generateMonths(startYear, startMonth) {
  const months = [];
  const now = new Date();
  let y = startYear, m = startMonth;
  while (y < now.getFullYear() || (y === now.getFullYear() && m <= now.getMonth() + 1)) {
    months.push(`${y}-${String(m).padStart(2, '0')}`);
    m++; if (m > 12) { m = 1; y++; }
  }
  return months;
}

function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }

main().catch(err => { console.error('Fatal:', err.message); process.exit(1); });
