'use strict';

const crypto = require('crypto');
const fs = require('fs');
const path = require('path');
const { StringDecoder } = require('string_decoder');

const { codexMessageFromEntry, codexRolloutIdFromPath, createCodexUserDeduper } = require('./session-history');
const { isProviderGeneratedUserContextText } = require('./provider-user-context');
const { sortTimelineMessages } = require('./timeline-order');

const PARSER_VERSION = 1;
const DEFAULT_CHUNK_BYTES = 1024 * 1024;
const DEFAULT_INITIAL_TAIL_BYTES = 2 * 1024 * 1024;
const MAX_VISIBLE_TEXT = 50 * 1024;

function sha1(value) {
  return crypto.createHash('sha1').update(String(value || '')).digest('hex');
}

function parseMetadataJson(value) {
  if (!value) return {};
  try {
    const parsed = JSON.parse(String(value));
    return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {};
  } catch {
    return {};
  }
}

function normalizeProvider(provider, filePath = '') {
  const raw = String(provider || '').trim().toLowerCase();
  if (raw) return raw;
  if (String(filePath || '').includes(`${path.sep}.codex${path.sep}sessions${path.sep}`)) return 'codex';
  return 'claude';
}

function sourceIdFromPath(filePath) {
  return codexRolloutIdFromPath(filePath || '')
    || path.basename(filePath || '').replace(/\.jsonl(\.bak)?$/, '');
}

function isSubagentTranscriptPath(filePath) {
  return String(filePath || '').replace(/\\/g, '/').includes('/subagents/');
}

function loadedTranscriptMessageFromRow(row) {
  const metadata = parseMetadataJson(row.metadata_json);
  const text = String(row.text || '');
  let role = row.role;
  let agentLabel = metadata.agentLabel || metadata.roleLabel || '';
  let roleLabel = metadata.roleLabel || metadata.agentLabel || '';
  const sourceKind = metadata.sourceKind || (isSubagentTranscriptPath(row.jsonl_path) ? 'subagent' : '');

  if (role === 'user' && isProviderGeneratedUserContextText(text)) return null;

  if (sourceKind === 'subagent') {
    metadata.sourceKind = 'subagent';
    if (role === 'user') {
      metadata.originalRole = metadata.originalRole || 'user';
      metadata.promptKind = metadata.promptKind || 'subagent';
      role = 'system';
      agentLabel = agentLabel || 'Subagent prompt';
      roleLabel = roleLabel || 'Subagent prompt';
    } else if (!agentLabel && role === 'assistant') {
      agentLabel = 'Subagent';
      roleLabel = roleLabel || 'Subagent';
    }
  }

  return {
    role,
    text,
    timestamp: row.timestamp || '',
    metadata,
    source: 'transcript-events',
    provider: row.provider || '',
    agentSessionId: row.agent_session_id,
    ctmSessionId: row.ctm_session_id || '',
    agentLabel: agentLabel || undefined,
    roleLabel: roleLabel || undefined,
  };
}

function ensureTranscriptTables(db, options = {}) {
  db.exec(`
    CREATE TABLE IF NOT EXISTS transcript_files (
      jsonl_path TEXT PRIMARY KEY,
      agent_session_id TEXT NOT NULL,
      ctm_session_id TEXT,
      provider TEXT,
      dev TEXT,
      ino TEXT,
      size INTEGER DEFAULT 0,
      mtime_ms REAL DEFAULT 0,
      offset INTEGER DEFAULT 0,
      partial_line TEXT DEFAULT '',
      parser_version INTEGER DEFAULT ${PARSER_VERSION},
      status TEXT DEFAULT 'ok',
      last_error TEXT DEFAULT '',
      updated_at TEXT DEFAULT CURRENT_TIMESTAMP
    );
    CREATE INDEX IF NOT EXISTS idx_transcript_files_agent
      ON transcript_files(agent_session_id);
    CREATE INDEX IF NOT EXISTS idx_transcript_files_ctm
      ON transcript_files(ctm_session_id);

    CREATE TABLE IF NOT EXISTS transcript_events (
      id INTEGER PRIMARY KEY AUTOINCREMENT,
      ctm_session_id TEXT,
      agent_session_id TEXT NOT NULL,
      provider TEXT,
      jsonl_path TEXT NOT NULL,
      seq INTEGER NOT NULL,
      role TEXT NOT NULL,
      timestamp TEXT,
      content TEXT NOT NULL,
      byte_start INTEGER DEFAULT 0,
      byte_end INTEGER DEFAULT 0,
      event_hash TEXT NOT NULL,
      metadata_json TEXT DEFAULT '{}',
      created_at TEXT DEFAULT CURRENT_TIMESTAMP,
      UNIQUE(agent_session_id, event_hash)
    );
    CREATE INDEX IF NOT EXISTS idx_transcript_events_agent_seq
      ON transcript_events(agent_session_id, seq);
    CREATE INDEX IF NOT EXISTS idx_transcript_events_ctm_seq
      ON transcript_events(ctm_session_id, seq);
    CREATE INDEX IF NOT EXISTS idx_transcript_events_time
      ON transcript_events(timestamp);
  `);

  // Phase 1 of the blob→rows migration: enrich transcript_events so it can be the
  // PRIMARY conversation store (not just a search overlay). Additive, idempotent —
  // SQLite has no ADD COLUMN IF NOT EXISTS, so each ALTER is guarded individually.
  //   turn_ordinal  – turn group (a user prompt + its answer/tool calls); assigned at
  //                   ingest, drives turn-mode pagination + render grouping.
  //   token_count   – cheap per-row token estimate; summed for the session total.
  //   uuid/parent_uuid – Claude message identity (promoted from metadata_json) for
  //                   streamed-partial dedup and parentUuid collapse parity.
  //   model/usage_json – per-message model + provider usage when present.
  //   content_json  – optional inline rich content for recent rows; empty rows
  //                   hydrate on demand from (jsonl_path, byte_start, byte_end).
  for (const ddl of [
    'ALTER TABLE transcript_events ADD COLUMN turn_ordinal INTEGER DEFAULT 0',
    'ALTER TABLE transcript_events ADD COLUMN token_count INTEGER DEFAULT 0',
    "ALTER TABLE transcript_events ADD COLUMN uuid TEXT DEFAULT ''",
    "ALTER TABLE transcript_events ADD COLUMN parent_uuid TEXT DEFAULT ''",
    "ALTER TABLE transcript_events ADD COLUMN model TEXT DEFAULT ''",
    "ALTER TABLE transcript_events ADD COLUMN usage_json TEXT DEFAULT ''",
    "ALTER TABLE transcript_events ADD COLUMN content_json TEXT DEFAULT ''",
  ]) {
    try { db.exec(ddl); } catch { /* column already exists */ }
  }
  try {
    db.exec(`CREATE INDEX IF NOT EXISTS idx_transcript_events_ctm_turn
      ON transcript_events(ctm_session_id, turn_ordinal, seq);`);
  } catch { /* index already exists / column missing on an old build */ }

  try {
    db.exec(`
      CREATE VIRTUAL TABLE IF NOT EXISTS transcript_events_fts USING fts5(
        content,
        content='', contentless_delete=1, tokenize='porter unicode61'
      );
    `);
    if (!options.skipFtsBackfill) {
      db.prepare(`
        INSERT INTO transcript_events_fts(rowid, content)
        SELECT e.id, e.content
        FROM transcript_events e
        LEFT JOIN transcript_events_fts fts ON fts.rowid = e.id
        WHERE fts.rowid IS NULL
          AND COALESCE(e.content, '') != ''
      `).run();
    }
  } catch (e) {
    // Some SQLite builds omit FTS5. Transcript rows remain queryable through the
    // row store; search simply skips the transcript-events source.
  }
}

function hasTranscriptEventsFts(db) {
  try {
    return !!db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='transcript_events_fts'").get();
  } catch {
    return false;
  }
}

function visibleTextFromClaudeContent(content) {
  if (typeof content === 'string') return content;
  if (!Array.isArray(content)) return '';
  return content
    .filter(block => block && block.type === 'text' && block.text)
    .map(block => block.text)
    .join('\n')
    .trim();
}

function cleanClaudeUserText(text) {
  const raw = String(text || '');
  const withoutReminders = raw.replace(/<system-reminder>[\s\S]*?<\/system-reminder>/g, '').trim();
  return withoutReminders || raw.trim();
}

function truncateVisibleText(text) {
  const value = String(text || '');
  if (value.length <= MAX_VISIBLE_TEXT) return value;
  return `${value.slice(0, MAX_VISIBLE_TEXT)}...truncated (${value.length} chars)`;
}

// Cheap per-row token estimate (~4 chars/token). Feeds an incremental session
// total; exact provider-authoritative counts are recomputed on the live path.
function estimateRowTokens(text) {
  const len = String(text || '').length;
  return len ? Math.max(1, Math.ceil(len / 4)) : 0;
}

function messageFromEntry(entry, provider, state = {}) {
  if (!entry || typeof entry !== 'object') return null;

  if (provider === 'codex') {
    if (entry.type === 'event_msg' && entry.payload?.type === 'task_complete' && entry.payload.last_agent_message) {
      const text = truncateVisibleText(String(entry.payload.last_agent_message || '').trim());
      return text ? {
        role: 'assistant',
        text,
        timestamp: entry.timestamp || '',
        metadata: { finalSource: 'task_complete', turnId: entry.payload.turn_id || '' },
      } : null;
    }
    const msg = codexMessageFromEntry(entry);
    if (!msg) return null;
    if (msg.role === 'user') {
      if (!state.codexUserDeduper) state.codexUserDeduper = createCodexUserDeduper();
      if (!state.codexUserDeduper.remember(msg.text, msg.timestamp)) return null;
    }
    return { role: msg.role, text: truncateVisibleText(msg.text), timestamp: msg.timestamp || '', metadata: {} };
  }

  if (entry.type === 'user' && entry.provider === 'walle' && typeof entry.content === 'string') {
    const text = truncateVisibleText(entry.content.trim());
    return text ? { role: 'user', text, timestamp: entry.timestamp || '', metadata: {} } : null;
  }

  if (entry.type === 'assistant' && entry.provider === 'walle' && typeof entry.content === 'string') {
    const text = truncateVisibleText(entry.content.trim());
    return text ? { role: 'assistant', text, timestamp: entry.timestamp || '', metadata: {} } : null;
  }

  if (entry.type === 'walle_part') {
    const data = entry.data || {};
    let text = '';
    if (entry.partType === 'tool_call') text = `[Tool: ${data.name || 'tool'}]`;
    else if (entry.partType === 'tool_result') text = `[Tool result: ${data.name || 'tool'}]`;
    else if (entry.partType === 'artifact') text = artifactText(data);
    else if (entry.partType === 'capability_routed') text = capabilityRouteText(data);
    else if (entry.partType === 'error') text = `[Wall-E error] ${data.message || ''}`.trim();
    else if (entry.partType === 'cancelled') text = '[Wall-E cancelled]';
    return text ? { role: 'system', text, timestamp: entry.timestamp || '', metadata: { partType: entry.partType || '' } } : null;
  }

  if (entry.type === 'user' && entry.message?.role === 'user') {
    const content = entry.message.content;
    const isToolResult = Array.isArray(content) && content.some(block => block && block.type === 'tool_result');
    let text = visibleTextFromClaudeContent(content);
    text = isToolResult ? text : cleanClaudeUserText(text);
    text = truncateVisibleText(text);
    if (!text) return null;
    if (!isToolResult && isProviderGeneratedUserContextText(text)) return null;
    const isSubagentTask = !isToolResult && state.sourceKind === 'subagent';
    return {
      role: isToolResult || isSubagentTask ? 'system' : 'user',
      text,
      timestamp: entry.timestamp || '',
      metadata: {
        uuid: entry.uuid || '',
        cwd: entry.cwd || '',
        gitBranch: entry.gitBranch || '',
        ...(isSubagentTask ? { sourceKind: 'subagent', originalRole: 'user', promptKind: 'subagent', roleLabel: 'Subagent prompt', agentLabel: 'Subagent prompt' } : {}),
      },
    };
  }

  if (entry.type === 'assistant' && entry.message?.role === 'assistant') {
    const text = truncateVisibleText(visibleTextFromClaudeContent(entry.message.content));
    if (!text) return null;
    return {
      role: 'assistant',
      text,
      timestamp: entry.timestamp || '',
      metadata: { parentUuid: entry.parentUuid || '', model: entry.message.model || entry.model || '' },
    };
  }

  return null;
}

function artifactText(data = {}) {
  const artifact = data.artifact || {};
  const kind = artifact.kind || data.type || 'artifact';
  const label = artifact.path || artifact.originalPath || artifact.artifactId || '';
  const bits = [`[Artifact: ${kind}]`];
  if (artifact.bytes) bits.push(`${artifact.bytes} bytes`);
  if (artifact.metadata?.page_count) bits.push(`${artifact.metadata.page_count} pages`);
  if (label) bits.push(label);
  return bits.join(' ');
}

function capabilityRouteText(data = {}) {
  const capabilities = Array.isArray(data.capabilities) ? data.capabilities : [];
  if (!capabilities.length) return '';
  const names = capabilities.map((capability) => capability.id || capability.label).filter(Boolean);
  return names.length ? `[Wall-E capability routes] ${names.join(', ')}` : '';
}

function readCompleteLines(filePath, { startOffset, maxBytes, partialLine, trimLeadingPartial }) {
  const fd = fs.openSync(filePath, 'r');
  const decoder = new StringDecoder('utf8');
  const chunkSize = DEFAULT_CHUNK_BYTES;
  const buf = Buffer.alloc(Math.min(chunkSize, Math.max(1, maxBytes || chunkSize)));
  let position = startOffset;
  let remaining = maxBytes;
  let carry = String(partialLine || '');
  let byteCursor = Math.max(0, startOffset - Buffer.byteLength(carry));
  const lines = [];
  let bytesReadTotal = 0;
  let leadingTrimmed = !trimLeadingPartial;

  try {
    while (remaining > 0) {
      const toRead = Math.min(buf.length, remaining);
      const bytesRead = fs.readSync(fd, buf, 0, toRead, position);
      if (bytesRead <= 0) break;
      bytesReadTotal += bytesRead;
      position += bytesRead;
      remaining -= bytesRead;

      let text = carry + decoder.write(buf.subarray(0, bytesRead));
      carry = '';

      if (!leadingTrimmed) {
        const nl = text.indexOf('\n');
        if (nl === -1) {
          byteCursor += Buffer.byteLength(text);
          continue;
        }
        const dropped = text.slice(0, nl + 1);
        byteCursor += Buffer.byteLength(dropped);
        text = text.slice(nl + 1);
        leadingTrimmed = true;
      }

      let start = 0;
      while (true) {
        const end = text.indexOf('\n', start);
        if (end === -1) break;
        const line = text.slice(start, end);
        const lineBytes = Buffer.byteLength(line) + 1;
        if (line.trim()) {
          lines.push({ line, byteStart: byteCursor, byteEnd: byteCursor + lineBytes });
        }
        byteCursor += lineBytes;
        start = end + 1;
      }
      carry = text.slice(start);
    }
    const tail = decoder.end();
    if (tail) carry += tail;
    return { lines, bytesRead: bytesReadTotal, nextOffset: position, partialLine: carry };
  } finally {
    fs.closeSync(fd);
  }
}

function sameFileIdentity(row, stat) {
  if (!row) return false;
  const dev = String(stat.dev ?? '');
  const ino = String(stat.ino ?? '');
  if (!row.dev || !row.ino) return true;
  return row.dev === dev && row.ino === ino;
}

function resolveStartCursor(row, stat, options) {
  const fileSize = Number(stat.size || 0);
  if (row && row.parser_version === PARSER_VERSION && sameFileIdentity(row, stat) && fileSize >= Number(row.offset || 0)) {
    return {
      offset: Number(row.offset || 0),
      partialLine: row.partial_line || '',
      trimLeadingPartial: false,
      resetReason: '',
    };
  }

  if (options.mode === 'tail' || options.initialTailBytes) {
    return {
      offset: Math.max(0, fileSize - Math.max(1, Number(options.initialTailBytes || DEFAULT_INITIAL_TAIL_BYTES))),
      partialLine: '',
      trimLeadingPartial: fileSize > Math.max(1, Number(options.initialTailBytes || DEFAULT_INITIAL_TAIL_BYTES)),
      resetReason: row ? 'identity-or-size-reset-tail' : 'cold-tail',
    };
  }

  return {
    offset: 0,
    partialLine: '',
    trimLeadingPartial: false,
    resetReason: row ? 'identity-or-size-reset-full' : 'cold-full',
  };
}

function eventHash(agentSessionId, message) {
  return sha1([
    agentSessionId,
    message.role || '',
    message.timestamp || '',
    message.text || '',
  ].join('\0'));
}

function ingestJsonlFile(db, options = {}) {
  if (!db) throw new Error('db is required');
  ensureTranscriptTables(db, { skipFtsBackfill: true });

  const filePath = path.resolve(String(options.filePath || ''));
  const stat = fs.statSync(filePath);
  const provider = normalizeProvider(options.provider, filePath);
  const agentSessionId = String(options.agentSessionId || path.basename(filePath).replace(/\.jsonl(\.bak)?$/, '')).trim();
  const ctmSessionId = String(options.ctmSessionId || agentSessionId).trim();
  if (!agentSessionId) throw new Error('agentSessionId is required');

  const existing = db.prepare('SELECT * FROM transcript_files WHERE jsonl_path = ?').get(filePath);
  const cursor = resolveStartCursor(existing, stat, options);
  if (stat.size <= cursor.offset && !cursor.partialLine) {
    db.prepare(`
      INSERT INTO transcript_files (jsonl_path, agent_session_id, ctm_session_id, provider, dev, ino, size, mtime_ms, offset, partial_line, parser_version, status, last_error, updated_at)
      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, '', ?, 'ok', '', datetime('now'))
      ON CONFLICT(jsonl_path) DO UPDATE SET
        agent_session_id=excluded.agent_session_id,
        ctm_session_id=excluded.ctm_session_id,
        provider=excluded.provider,
        dev=excluded.dev,
        ino=excluded.ino,
        size=excluded.size,
        mtime_ms=excluded.mtime_ms,
        parser_version=excluded.parser_version,
        status='ok',
        last_error='',
        updated_at=datetime('now')
    `).run(filePath, agentSessionId, ctmSessionId, provider, String(stat.dev ?? ''), String(stat.ino ?? ''), stat.size, stat.mtimeMs, cursor.offset, PARSER_VERSION);
    return { inserted: 0, parsed: 0, bytesRead: 0, offset: cursor.offset, partialLine: '', skipped: true };
  }

  const maxBytes = Math.min(
    Math.max(0, stat.size - cursor.offset),
    Math.max(1, Number(options.maxBytes || (stat.size - cursor.offset)))
  );
  const read = readCompleteLines(filePath, {
    startOffset: cursor.offset,
    maxBytes,
    partialLine: cursor.partialLine,
    trimLeadingPartial: cursor.trimLeadingPartial,
  });

  const state = {
    codexUserDeduper: createCodexUserDeduper(),
    sourceFile: filePath,
    sourceKind: isSubagentTranscriptPath(filePath) ? 'subagent' : '',
  };
  const messages = [];
  for (const lineInfo of read.lines) {
    let entry;
    try { entry = JSON.parse(lineInfo.line); } catch { continue; }
    const message = messageFromEntry(entry, provider, state);
    if (!message || !message.text) continue;
    messages.push({ ...message, byteStart: lineInfo.byteStart, byteEnd: lineInfo.byteEnd });
  }

  // Seed seq AND turn_ordinal from the durable max so a restart mid-session never
  // resets either (crash-safe resume — turns must stay monotonic across boots).
  const nextSeqRow = db.prepare('SELECT COALESCE(MAX(seq) + 1, 0) AS nextSeq, COALESCE(MAX(turn_ordinal), 0) AS maxTurn FROM transcript_events WHERE agent_session_id = ?').get(agentSessionId);
  let seq = Number(nextSeqRow?.nextSeq || 0);
  let turn = Number(nextSeqRow?.maxTurn || 0);
  let inserted = 0;
  const insertEvent = db.prepare(`
    INSERT OR IGNORE INTO transcript_events
      (ctm_session_id, agent_session_id, provider, jsonl_path, seq, role, timestamp, content, byte_start, byte_end, event_hash, metadata_json,
       turn_ordinal, token_count, uuid, parent_uuid, model, usage_json, content_json)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
  `);
  const insertEventFts = hasTranscriptEventsFts(db)
    ? db.prepare('INSERT INTO transcript_events_fts(rowid, content) VALUES (?, ?)')
    : null;
  const upsertFile = db.prepare(`
    INSERT INTO transcript_files
      (jsonl_path, agent_session_id, ctm_session_id, provider, dev, ino, size, mtime_ms, offset, partial_line, parser_version, status, last_error, updated_at)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'ok', '', datetime('now'))
    ON CONFLICT(jsonl_path) DO UPDATE SET
      agent_session_id=excluded.agent_session_id,
      ctm_session_id=excluded.ctm_session_id,
      provider=excluded.provider,
      dev=excluded.dev,
      ino=excluded.ino,
      size=excluded.size,
      mtime_ms=excluded.mtime_ms,
      offset=excluded.offset,
      partial_line=excluded.partial_line,
      parser_version=excluded.parser_version,
      status='ok',
      last_error='',
      updated_at=datetime('now')
  `);

  // Insert in bounded sub-transactions so a large catch-up ingest never holds the SQLite
  // write lock for the whole file. One whole-file db.transaction() was observed holding the
  // lock ~20s on the primary, bloating the WAL (no checkpoint can run mid-transaction) →
  // reader/read-pool requests time out → callers fall back to the main thread (the bursty
  // freezes). Each chunk is its own transaction = its own lock acquire/release, so
  // checkpoints and readers interleave and the WAL stays bounded. INSERT OR IGNORE
  // (event_hash) keeps re-runs idempotent, so the file cursor is advanced once at the end —
  // a crash mid-ingest simply re-processes from the old offset without duplicating rows.
  const CHUNK = Math.max(25, Math.min(5000, Number(options.ingestChunkSize || process.env.CTM_TRANSCRIPT_INGEST_CHUNK || 250)));
  const insertChunk = db.transaction((slice) => {
    for (const message of slice) {
      // A turn = one real user prompt + the assistant answer(s)/tool calls that
      // follow it. messageFromEntry already maps tool-results and subagent prompts
      // to role 'system', so role==='user' is a genuine prompt → start a new turn.
      // Assistant/system rows inherit the current turn. (Read-side turn grouping in
      // Phase 3 keys on this; it mirrors the renderer's user-message turn boundary.)
      if (message.role === 'user') turn++;
      const meta = message.metadata || {};
      const hash = eventHash(agentSessionId, message);
      const info = insertEvent.run(
        ctmSessionId,
        agentSessionId,
        provider,
        filePath,
        seq,
        message.role,
        message.timestamp || '',
        message.text,
        message.byteStart || 0,
        message.byteEnd || 0,
        hash,
        JSON.stringify(meta),
        turn,
        estimateRowTokens(message.text),
        String(meta.uuid || ''),
        String(meta.parentUuid || ''),
        String(meta.model || ''),
        '',  // usage_json: enriched later; empty keeps the row small
        ''   // content_json: rich content hydrated on demand from byte offsets
      );
      if (info.changes > 0) {
        if (insertEventFts) {
          try { insertEventFts.run(info.lastInsertRowid, message.text); } catch {}
        }
        inserted++;
        seq++;
      }
    }
  });
  for (let i = 0; i < messages.length; i += CHUNK) {
    insertChunk(messages.slice(i, i + CHUNK));
  }
  // Advance the file cursor only after all events are committed. INSERT OR IGNORE makes a
  // re-run from the old offset idempotent, so a crash between chunks loses no data.
  upsertFile.run(
    filePath,
    agentSessionId,
    ctmSessionId,
    provider,
    String(stat.dev ?? ''),
    String(stat.ino ?? ''),
    stat.size,
    stat.mtimeMs,
    read.nextOffset,
    read.partialLine || '',
    PARSER_VERSION
  );

  return {
    inserted,
    parsed: messages.length,
    bytesRead: read.bytesRead,
    offset: read.nextOffset,
    partialLine: read.partialLine || '',
    resetReason: cursor.resetReason,
    messages,
  };
}

function ftsSearchTranscriptEvents(db, query, limit = 50, options = {}) {
  ensureTranscriptTables(db, { skipFtsBackfill: true });
  if (!hasTranscriptEventsFts(db)) return [];
  try {
    const { expandQueryFuzzy } = require('../fuzzy-utils');
    const { ftsQuery, expansions } = expandQueryFuzzy(query, options);
    if (!ftsQuery) return [];
    return db.prepare(`
      SELECT
        e.id,
        e.ctm_session_id,
        e.agent_session_id,
        e.provider,
        e.role,
        e.timestamp,
        e.created_at,
        e.content,
        e.jsonl_path,
        f.size AS file_size,
        highlight(transcript_events_fts, 0, '<mark>', '</mark>') AS h_content
      FROM transcript_events_fts fts
      JOIN transcript_events e ON e.id = fts.rowid
      LEFT JOIN transcript_files f ON f.jsonl_path = e.jsonl_path
      WHERE transcript_events_fts MATCH ?
      ORDER BY fts.rank
      LIMIT ?
    `).all(ftsQuery, Math.max(1, Number(limit || 50))).map(row => ({
      ...row,
      h_content: row.h_content || row.content,
      session_id: row.ctm_session_id || row.agent_session_id,
      _fuzzyExpansions: expansions.length > 0 ? expansions : undefined,
    }));
  } catch (e) {
    if (!String(e.message || '').includes('no such table')) {
      console.error('[transcript-store] Transcript FTS search error:', e.message);
    }
    return [];
  }
}

function transcriptSourceIds(db, sessionId, extraIds = []) {
  ensureTranscriptTables(db, { skipFtsBackfill: true });
  const ids = new Set();
  const add = value => {
    const clean = String(value || '').trim();
    if (clean) ids.add(clean);
  };
  add(sessionId);
  for (const id of extraIds || []) add(id);
  try {
    const rows = db.prepare(
      'SELECT agent_session_id, ctm_session_id FROM transcript_files WHERE ctm_session_id = ? OR agent_session_id = ?'
    ).all(sessionId, sessionId);
    for (const row of rows) {
      add(row.agent_session_id);
      add(row.ctm_session_id);
    }
  } catch {}
  return [...ids];
}

function loadTranscriptMessages(db, ids = []) {
  ensureTranscriptTables(db, { skipFtsBackfill: true });
  const cleanIds = [...new Set((ids || []).map(id => String(id || '').trim()).filter(Boolean))];
  if (!cleanIds.length) return [];
  const placeholders = cleanIds.map(() => '?').join(',');
  return db.prepare(`
    SELECT ctm_session_id, agent_session_id, provider, role, content AS text, timestamp, seq, jsonl_path, metadata_json
      FROM transcript_events
     WHERE ctm_session_id IN (${placeholders})
        OR agent_session_id IN (${placeholders})
     ORDER BY
       CASE WHEN timestamp IS NULL OR timestamp = '' THEN 1 ELSE 0 END,
       timestamp,
       agent_session_id,
       seq
  `).all(...cleanIds, ...cleanIds)
    .map(loadedTranscriptMessageFromRow)
    .filter(Boolean);
}

// ---------------------------------------------------------------------------
// Phase 3: keyset/offset-paginated reads over transcript_events, so a session's
// conversation is served from per-message ROWS (≤ limit materialized) instead of
// JSON.parse-ing a multi-GB blob. Reproduces lib/message-pagination.js semantics
// (newest-first offset; chronological within a page) so the API/response shape and
// the renderer are unchanged. Page reads are O(limit) on the (ctm_session_id, seq)
// / (ctm_session_id, turn_ordinal) indexes — the 28-33s blob-parse freeze is gone.
// ---------------------------------------------------------------------------

function _resolveSessionRowIds(db, ids) {
  const list = Array.isArray(ids) ? ids : [ids];
  const seed = list.find(Boolean) || '';
  return transcriptSourceIds(db, seed, list);
}

// True only when the row store can serve this session: the migrated columns exist
// AND at least one row is present for the id set. Callers fall back to the legacy
// blob/JSONL ladder otherwise, so a half-migrated session never breaks.
function sessionMessagesRowsAvailable(db, ids) {
  try {
    ensureTranscriptTables(db, { skipFtsBackfill: true });
    const cols = db.prepare("PRAGMA table_info(transcript_events)").all().map(c => c.name);
    if (!cols.includes('turn_ordinal')) return false;
    const sids = _resolveSessionRowIds(db, ids);
    if (!sids.length) return false;
    const ph = sids.map(() => '?').join(',');
    const row = db.prepare(`SELECT 1 FROM transcript_events WHERE ctm_session_id IN (${ph}) OR agent_session_id IN (${ph}) LIMIT 1`).get(...sids, ...sids);
    if (!row) return false;
    // FRESHNESS GATE: the rows must be CAUGHT UP to every source file. Stat the LIVE file
    // (not transcript_files.size, which is itself only as fresh as the last ingest): if the
    // file has grown past the durable ingest cursor (offset), the rows are missing its recent
    // tail — serving them would show a stale conversation (caught a real case on a giant Codex
    // session whose rows lagged the file by ~10h). When stale, return false so the caller uses
    // the legacy blob/JSONL path, which reads the live file and is always complete. 64KB
    // tolerates the trailing partial line; anything more is real lag.
    const files = db.prepare(
      `SELECT jsonl_path, offset FROM transcript_files WHERE ctm_session_id IN (${ph}) OR agent_session_id IN (${ph})`
    ).all(...sids, ...sids);
    if (!files.length) return false;
    for (const f of files) {
      let size = 0;
      try { size = fs.statSync(f.jsonl_path).size; } catch { return false; } // file gone → use legacy path
      if (size - Number(f.offset || 0) > 65536) return false;
    }
    return true;
  } catch { return false; }
}

function _whereIds(sids) {
  const ph = sids.map(() => '?').join(',');
  return { clause: `(ctm_session_id IN (${ph}) OR agent_session_id IN (${ph}))`, args: [...sids, ...sids] };
}

const _ROW_ORDER = `CASE WHEN timestamp IS NULL OR timestamp = '' THEN 1 ELSE 0 END, timestamp, agent_session_id, seq`;
const _ROW_COLS = `ctm_session_id, agent_session_id, provider, role, content AS text, timestamp, seq, jsonl_path, byte_start, byte_end, metadata_json, turn_ordinal`;

function countSessionMessages(db, ids) {
  const sids = _resolveSessionRowIds(db, ids);
  if (!sids.length) return 0;
  const w = _whereIds(sids);
  return Number(db.prepare(`SELECT COUNT(*) AS c FROM transcript_events WHERE ${w.clause}`).get(...w.args)?.c || 0);
}

// Newest-first offset (offset 0 = the most recent `limit`), chronological within the
// page — matches lib/message-pagination.js paginateMessages().
function getSessionMessagesPage(db, ids, { offset = 0, limit = 200 } = {}) {
  ensureTranscriptTables(db, { skipFtsBackfill: true });
  const sids = _resolveSessionRowIds(db, ids);
  if (!sids.length) return { messages: [], total: 0, has_more: false, next_offset: Number(offset) || 0 };
  const w = _whereIds(sids);
  const total = Number(db.prepare(`SELECT COUNT(*) AS c FROM transcript_events WHERE ${w.clause}`).get(...w.args)?.c || 0);
  const lim = Math.max(1, Math.min(1000, Number(limit) || 200));
  const off = Math.max(0, Number(offset) || 0);
  const start = Math.max(0, total - off - lim);
  const take = Math.max(0, Math.min(lim, total - off - start));
  const raw = take === 0 ? [] : db.prepare(
    `SELECT ${_ROW_COLS} FROM transcript_events WHERE ${w.clause} ORDER BY ${_ROW_ORDER} LIMIT ? OFFSET ?`
  ).all(...w.args, take, start);
  const messages = raw.map(loadedTranscriptMessageFromRow).filter(Boolean);
  return { messages, total, has_more: start > 0, next_offset: off + take, _rows: raw };
}

// Newest-first by turn (offset 0 = the most recent `limit` turns). Returns all rows
// belonging to the page's turns in chronological order — matches paginateMessageTurns().
function getSessionTurnsPage(db, ids, { offset = 0, limit = 50 } = {}) {
  ensureTranscriptTables(db, { skipFtsBackfill: true });
  const sids = _resolveSessionRowIds(db, ids);
  const empty = { messages: [], total: 0, has_more: false, next_offset: Number(offset) || 0, page_kind: 'turns', turn_count: 0 };
  if (!sids.length) return empty;
  const w = _whereIds(sids);
  const turns = db.prepare(`SELECT DISTINCT turn_ordinal FROM transcript_events WHERE ${w.clause} ORDER BY turn_ordinal`).all(...w.args).map(r => Number(r.turn_ordinal));
  const totalTurns = turns.length;
  const lim = Math.max(1, Math.min(1000, Number(limit) || 50));
  const off = Math.max(0, Number(offset) || 0);
  const startIdx = Math.max(0, totalTurns - off - lim);
  const endIdx = Math.max(startIdx, totalTurns - off);
  const pageTurns = turns.slice(startIdx, endIdx);
  if (!pageTurns.length) return { ...empty, total: totalTurns, has_more: startIdx > 0 };
  const minT = pageTurns[0], maxT = pageTurns[pageTurns.length - 1];
  const raw = db.prepare(
    `SELECT ${_ROW_COLS} FROM transcript_events WHERE ${w.clause} AND turn_ordinal BETWEEN ? AND ? ORDER BY ${_ROW_ORDER}`
  ).all(...w.args, minT, maxT);
  const messages = raw.map(loadedTranscriptMessageFromRow).filter(Boolean);
  return {
    messages, total: totalTurns, has_more: startIdx > 0, next_offset: off + pageTurns.length,
    page_kind: 'turns', turn_count: pageTurns.length, first_turn_index: minT, last_turn_index: maxT, _rows: raw,
  };
}

// Phase 4 (existing-user backfill): enrich rows that were ingested BEFORE Phase 2 and
// therefore have turn_ordinal=0 / token_count=0. Walks one agent session's rows in order
// and assigns turn_ordinal (++ on each real user prompt) + a token estimate, in bounded
// UPDATE transactions. Idempotent: only runs for a session that has user rows but no turns
// yet (after it runs, MAX(turn_ordinal) > 0, so it never repeats). Returns rows updated.
function enrichTranscriptRowsForSession(db, agentSessionId) {
  ensureTranscriptTables(db, { skipFtsBackfill: true });
  const aid = String(agentSessionId || '').trim();
  if (!aid) return 0;
  const stat = db.prepare(`SELECT MAX(turn_ordinal) AS maxTurn,
    SUM(CASE WHEN role = 'user' THEN 1 ELSE 0 END) AS users
    FROM transcript_events WHERE agent_session_id = ?`).get(aid);
  if (!stat || Number(stat.maxTurn || 0) > 0 || Number(stat.users || 0) === 0) return 0;
  const rows = db.prepare('SELECT id, role, content FROM transcript_events WHERE agent_session_id = ? ORDER BY seq').all(aid);
  const upd = db.prepare('UPDATE transcript_events SET turn_ordinal = ?, token_count = ? WHERE id = ?');
  let turn = 0, n = 0;
  const tx = db.transaction((batch) => {
    for (const r of batch) {
      if (r.role === 'user') turn++;
      upd.run(turn, estimateRowTokens(r.content), r.id);
      n++;
    }
  });
  const CHUNK = 500;
  for (let i = 0; i < rows.length; i += CHUNK) tx(rows.slice(i, i + CHUNK));
  return n;
}

// Find agent sessions whose rows still need enrichment (have user rows, no turns yet),
// bounded — for a background sweep that migrates existing users without a big-bang.
function findUnenrichedSessions(db, limit = 25) {
  ensureTranscriptTables(db, { skipFtsBackfill: true });
  try {
    return db.prepare(`SELECT agent_session_id FROM transcript_events
      GROUP BY agent_session_id
      HAVING MAX(turn_ordinal) = 0 AND SUM(CASE WHEN role = 'user' THEN 1 ELSE 0 END) > 0
      LIMIT ?`).all(Math.max(1, Math.min(500, Number(limit) || 25))).map(r => r.agent_session_id);
  } catch { return []; }
}

// Session token total from per-row estimates (no full-array rebuild).
function getSessionTokenSum(db, ids) {
  const sids = _resolveSessionRowIds(db, ids);
  if (!sids.length) return 0;
  const w = _whereIds(sids);
  return Number(db.prepare(`SELECT COALESCE(SUM(token_count), 0) AS t FROM transcript_events WHERE ${w.clause}`).get(...w.args)?.t || 0);
}

function mergeMessageLists(baseMessages, overlayMessages) {
  const merged = [];
  const seen = new Set();
  const add = (message) => {
    const role = String(message?.role || '').trim();
    const text = String(message?.text ?? message?.content ?? message?.message ?? '').trim();
    if (!role || !text) return;
    const timestamp = String(message?.timestamp || message?.created_at || message?.createdAt || '');
    const key = [role, timestamp, sha1(text)].join('\0');
    if (seen.has(key)) return;
    seen.add(key);
    merged.push({ ...message, role, text, timestamp });
  };
  for (const message of Array.isArray(baseMessages) ? baseMessages : []) add(message);
  for (const message of Array.isArray(overlayMessages) ? overlayMessages : []) add(message);
  return sortTimelineMessages(merged);
}

function catchUpTranscriptFiles(db, options = {}) {
  ensureTranscriptTables(db, { skipFtsBackfill: true });
  const sessionId = String(options.sessionId || '').trim();
  const filePaths = Array.isArray(options.filePaths) ? options.filePaths : [];
  const maxBytesPerFile = Math.max(1, Number(options.maxBytesPerFile || DEFAULT_INITIAL_TAIL_BYTES));
  const initialTailBytes = Math.max(1, Number(options.initialTailBytes || DEFAULT_INITIAL_TAIL_BYTES));
  const largeFileThresholdBytes = Math.max(1, Number(options.largeFileThresholdBytes || initialTailBytes));
  const providerByAgentId = options.providerByAgentId || {};
  const seenPaths = new Set();
  const agentSessionIds = new Set();
  let inserted = 0;
  let parsed = 0;
  let bytesRead = 0;
  let failed = 0;
  let tailedColdFiles = 0;

  for (const rawPath of filePaths) {
    const filePath = path.resolve(String(rawPath || ''));
    if (!filePath || seenPaths.has(filePath)) continue;
    seenPaths.add(filePath);
    let stat;
    try { stat = fs.statSync(filePath); } catch { continue; }
    if (!stat.isFile()) continue;

    const agentSessionId = sourceIdFromPath(filePath);
    if (!agentSessionId) continue;
    agentSessionIds.add(agentSessionId);
    const provider = normalizeProvider(providerByAgentId[agentSessionId], filePath);
    let cursor = null;
    try {
      cursor = db.prepare('SELECT offset FROM transcript_files WHERE jsonl_path = ?').get(filePath);
    } catch {}
    const hasCursor = cursor && Number(cursor.offset || 0) > 0;
    const isLargeColdFile = !hasCursor && Number(stat.size || 0) >= largeFileThresholdBytes;

    try {
      const result = ingestJsonlFile(db, {
        filePath,
        agentSessionId,
        ctmSessionId: sessionId || agentSessionId,
        provider,
        mode: isLargeColdFile ? 'tail' : undefined,
        initialTailBytes,
        maxBytes: hasCursor ? maxBytesPerFile : (isLargeColdFile ? initialTailBytes : Math.min(stat.size, maxBytesPerFile)),
      });
      inserted += result.inserted || 0;
      parsed += result.parsed || 0;
      bytesRead += result.bytesRead || 0;
      if (isLargeColdFile) tailedColdFiles += 1;
    } catch (e) {
      failed += 1;
      try { markTranscriptFileError(db, filePath, e); } catch {}
    }
  }

  return {
    inserted,
    parsed,
    bytesRead,
    failed,
    tailedColdFiles,
    agentSessionIds: [...agentSessionIds],
  };
}

function markTranscriptFileError(db, filePath, error) {
  ensureTranscriptTables(db, { skipFtsBackfill: true });
  const resolved = path.resolve(String(filePath || ''));
  db.prepare(`
    INSERT INTO transcript_files (jsonl_path, agent_session_id, status, last_error, updated_at)
    VALUES (?, '', 'error', ?, datetime('now'))
    ON CONFLICT(jsonl_path) DO UPDATE SET
      status='error',
      last_error=excluded.last_error,
      updated_at=datetime('now')
  `).run(resolved, String(error?.message || error || '').slice(0, 500));
}

module.exports = {
  PARSER_VERSION,
  DEFAULT_INITIAL_TAIL_BYTES,
  ensureTranscriptTables,
  ftsSearchTranscriptEvents,
  ingestJsonlFile,
  loadTranscriptMessages,
  catchUpTranscriptFiles,
  markTranscriptFileError,
  mergeMessageLists,
  messageFromEntry,
  isSubagentTranscriptPath,
  normalizeProvider,
  sourceIdFromPath,
  transcriptSourceIds,
  // Phase 3 row-store readers
  sessionMessagesRowsAvailable,
  countSessionMessages,
  getSessionMessagesPage,
  getSessionTurnsPage,
  getSessionTokenSum,
  // Phase 4 existing-user enrichment backfill
  enrichTranscriptRowsForSession,
  findUnenrichedSessions,
};
