'use strict';

const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const crypto = require('node:crypto');

const DEFAULT_SEARCH_LIMIT = 10;
const MAX_SEARCH_LIMIT = 50;
const DEFAULT_CONTEXT_LIMIT = 200;
const MAX_CONTEXT_LIMIT = 5000;
const DEFAULT_TOKEN_BUDGET = 12000;
const CTM_SESSION_TABLES = [
  'session_conversations',
  'session_messages',
  'session_messages_fts',
  'ctm_sessions',
  'agent_sessions',
];
let Database = null;

function requireBetterSqlite3() {
  try {
    return require('better-sqlite3');
  } catch (err) {
    if (err?.code !== 'MODULE_NOT_FOUND' || !String(err.message || '').includes('better-sqlite3')) {
      throw err;
    }
    // CTM imports this module directly for session-memory APIs. In that mode
    // only claude-task-manager dependencies may be installed.
    return require(path.join(__dirname, '..', '..', 'claude-task-manager', 'node_modules', 'better-sqlite3'));
  }
}

function loadBetterSqlite3() {
  if (!Database) Database = requireBetterSqlite3();
  return Database;
}

function assertDirectReadAllowed(options = {}) {
  const env = options.env || process.env;
  if (!env.CTM_PROCESS_ROLE || env.CTM_ALLOW_DIRECT_SESSION_CONTEXT_DB === '1') return;
  const err = new Error(
    'CTM session context direct SQLite read is disabled inside CTM_PROCESS_ROLE; pass the CTM-owned db handle instead.'
  );
  err.code = 'CTM_DIRECT_DB_READ_BLOCKED';
  throw err;
}

function resolveCtmDbPath(env = process.env) {
  if (env.CTM_DB_PATH) return env.CTM_DB_PATH;
  if (env.CTM_DATA_DIR) return path.join(env.CTM_DATA_DIR, 'task-manager.db');
  if (env.WALLE_DEV_DIR) return path.join(env.WALLE_DEV_DIR, 'task-manager.db');
  return path.join(env.HOME || os.homedir(), '.walle', 'data', 'task-manager.db');
}

function resolveCtmDbCandidatePaths(env = process.env) {
  const candidates = [];
  const push = (dbPath, source) => {
    if (!dbPath) return;
    const resolved = path.resolve(dbPath);
    if (candidates.some((candidate) => candidate.dbPath === resolved)) return;
    candidates.push({ dbPath: resolved, source });
  };

  push(env.CTM_DB_PATH, 'configured-ctm-db');
  if (env.CTM_DATA_DIR) push(path.join(env.CTM_DATA_DIR, 'task-manager.db'), 'ctm-data-dir');
  if (env.WALLE_DEV_DIR) push(path.join(env.WALLE_DEV_DIR, 'task-manager.db'), 'walle-dev-db');
  push(path.join(env.HOME || os.homedir(), '.walle', 'data', 'task-manager.db'), 'wall-e-cache');
  if (!candidates.length) push(resolveCtmDbPath(env), 'ctm-db');
  return candidates;
}

function openCtmDb({ dbPath, readonly = true, busyTimeoutMs = 2500, env = process.env } = {}) {
  assertDirectReadAllowed({ env });
  const resolved = path.resolve(dbPath || resolveCtmDbPath(env));
  if (!fs.existsSync(resolved)) {
    return { db: null, dbPath: resolved, close: () => {}, unavailable: true };
  }
  const SqliteDatabase = loadBetterSqlite3();
  const db = new SqliteDatabase(resolved, { readonly, fileMustExist: true });
  const timeout = Number.isFinite(Number(busyTimeoutMs)) ? Math.max(0, Math.trunc(Number(busyTimeoutMs))) : 2500;
  db.pragma(`busy_timeout = ${timeout}`);
  if (readonly) {
    db.pragma('query_only = ON');
  }
  return { db, dbPath: resolved, readonly: Boolean(readonly), close: () => db.close(), unavailable: false };
}

function withCtmDb(options, fn) {
  if (options?.db) {
    return fn(options.db, { dbPath: options.dbPath || '', close: () => {}, owned: false });
  }

  const attempts = [];
  const candidates = options?.dbPath
    ? [{ dbPath: path.resolve(options.dbPath), source: 'explicit-db-path' }]
    : resolveCtmDbCandidatePaths(options?.env || process.env);

  let lastFailure = null;
  for (let i = 0; i < candidates.length; i += 1) {
    const candidate = candidates[i];
    let handle;
    try {
      handle = openCtmDb({ ...options, dbPath: candidate.dbPath });
    } catch (err) {
      const classified = classifySqliteFailure(err, 'ctm_db_open_failed');
      const failure = {
        db_path: candidate.dbPath,
        active_source: candidate.source,
        ok: false,
        reason: classified.reason,
        error: err?.message || String(err),
      };
      attempts.push(failure);
      lastFailure = failure;
      continue;
    }

    if (!handle.db) {
      const failure = {
        db_path: handle.dbPath,
        active_source: candidate.source,
        ok: false,
        unavailable: true,
        reason: 'ctm_db_not_found',
      };
      attempts.push(failure);
      lastFailure = failure;
      continue;
    }

    try {
      const result = fn(handle.db, {
        dbPath: handle.dbPath,
        activeSource: candidate.source,
        close: handle.close,
        owned: true,
      });
      const annotated = annotateDbResult(result, {
        dbPath: handle.dbPath,
        activeSource: candidate.source,
        attempts,
        fallbackUsed: i > 0,
      });
      attempts.push(attemptFromResult(annotated));
      if (!isDbSourceFailure(annotated) || options?.fallback === false) {
        return annotated;
      }
      lastFailure = annotated;
    } finally {
      handle.close();
    }
  }

  return {
    ok: false,
    source: 'ctm-db',
    unavailable: true,
    db_path: lastFailure?.db_path || candidates[0]?.dbPath || '',
    active_source: lastFailure?.active_source || candidates[0]?.source || 'ctm-db',
    fallback_used: attempts.length > 1,
    attempts,
    reason: lastFailure?.reason || 'ctm_db_not_found',
    error: lastFailure?.error || '',
  };
}

function annotateDbResult(result, { dbPath, activeSource, attempts, fallbackUsed }) {
  if (!result || typeof result !== 'object') return result;
  return {
    ...result,
    db_path: result.db_path || dbPath,
    active_source: result.active_source || activeSource,
    fallback_used: Boolean(result.fallback_used || fallbackUsed || attempts.length > 0),
    attempts: attempts.length ? [...attempts] : result.attempts,
  };
}

function attemptFromResult(result) {
  return {
    db_path: result?.db_path || '',
    active_source: result?.active_source || '',
    ok: Boolean(result?.ok),
    reason: result?.reason || '',
    error: result?.error || '',
    tables: result?.tables || result?.cached_tables || [],
    missing_tables: result?.missing_tables || [],
  };
}

function isDbSourceFailure(result) {
  if (!result || result.ok !== false) return false;
  return [
    'ctm_db_not_found',
    'ctm_db_open_failed',
    'ctm_db_locked',
    'ctm_db_schema_read_failed',
    'ctm_session_tables_missing',
  ].includes(result.reason);
}

function classifySqliteFailure(err, fallbackReason) {
  const code = String(err?.code || '');
  const message = String(err?.message || err || '');
  if (code === 'CTM_DIRECT_DB_READ_BLOCKED') {
    return { reason: 'ctm_db_direct_read_blocked' };
  }
  if (/SQLITE_(BUSY|LOCKED)/.test(code) || /database is locked|database table is locked|busy/i.test(message)) {
    return { reason: 'ctm_db_locked' };
  }
  return { reason: fallbackReason };
}

function hasOpenableCtmDb(env = process.env) {
  return resolveCtmDbCandidatePaths(env).some((candidate) => fs.existsSync(candidate.dbPath));
}

function tableRowCount(db, table) {
  if (!hasTable(db, table)) return null;
  try {
    return db.prepare(`SELECT COUNT(*) AS count FROM ${quoteIdent(table)}`).get().count;
  } catch {
    return null;
  }
}

function searchCtmSessions({ query, limit = DEFAULT_SEARCH_LIMIT, db, dbPath, env } = {}) {
  const cleanQuery = String(query || '').trim();
  if (!cleanQuery) return { ok: false, source: 'ctm-db', results: [], reason: 'query_required' };
  const requestedLimit = clampLimit(limit, DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT);

  return withCtmDb({ db, dbPath, env }, (conn, handle) => {
    const tableStatus = inspectSessionTables(conn);
    if (!tableStatus.ok) {
      return {
        ok: false,
        source: 'ctm-db',
        db_path: handle.dbPath,
        results: [],
        ...tableStatus,
      };
    }

    const rawResults = [];
    rawResults.push(...searchSessionMessagesFts(conn, cleanQuery, requestedLimit * 4));
    rawResults.push(...searchSessionMessagesLike(conn, cleanQuery, requestedLimit * 4));
    rawResults.push(...searchSessionConversationMessages(conn, cleanQuery, requestedLimit * 6));
    rawResults.push(...searchSessionMetadata(conn, cleanQuery, requestedLimit * 2));

    const results = dedupeItems(rawResults, {
      limit: requestedLimit,
      contentFields: ['snippet', 'content'],
      keyPrefix: (item) => `${item.session_id || ''}:${item.role || ''}`,
    });

    return {
      ok: true,
      source: 'ctm-db',
      db_path: handle.dbPath,
      active_source: handle.activeSource || 'ctm-db',
      query: cleanQuery,
      count: results.length,
      results,
    };
  });
}

function getCtmSessionContext({
  session_id,
  source_id,
  session_ids,
  ids,
  limit = DEFAULT_CONTEXT_LIMIT,
  cursor = 0,
  include_raw = false,
  dedupe = true,
  format = 'messages',
  db,
  dbPath,
  env,
} = {}) {
  const identifiers = normalizeIdentifiers(session_ids || ids || session_id || source_id);
  if (identifiers.length === 0) {
    return { ok: false, source: 'ctm-db', reason: 'session_id_required', sessions: [], messages: [] };
  }

  return withCtmDb({ db, dbPath, env }, (conn, handle) => {
    const tableStatus = inspectSessionTables(conn);
    if (!tableStatus.ok) {
      return {
        ok: false,
        source: 'ctm-db',
        db_path: handle.dbPath,
        sessions: [],
        messages: [],
        ...tableStatus,
      };
    }

    const bundles = identifiers.map((id) => resolveSessionBundle(conn, id)).filter(Boolean);
    const seenBundles = new Set();
    const uniqueBundles = bundles.filter((bundle) => {
      const key = bundle.ctm_session_id || bundle.requested_id;
      if (seenBundles.has(key)) return false;
      seenBundles.add(key);
      return true;
    });

    const sessions = [];
    const allMessages = [];
    let servedFromRows = false;
    for (const bundle of uniqueBundles) {
      const conversations = getConversationRows(conn, bundle);
      const sessionMessages = [];
      for (const row of conversations) {
        // Phase 5: prefer the faithful per-message rows; fall back to the blob when absent.
        const rowMessages = readSessionMessageRows(conn, row._conversation_id);
        const messages = rowMessages || parseConversationMessages(row);
        if (rowMessages) servedFromRows = true;
        for (let i = 0; i < messages.length; i += 1) {
          const normalized = normalizeMessage(messages[i], {
            row,
            bundle,
            index: i,
            includeRaw: include_raw,
          });
          if (normalized.content || include_raw) sessionMessages.push(normalized);
        }
      }

      const dedupedSessionMessages = dedupe ? dedupeMessages(sessionMessages) : sessionMessages;
      sessions.push(formatSessionSummary(bundle, conversations, dedupedSessionMessages));
      allMessages.push(...dedupedSessionMessages);
    }

    const orderedMessages = orderMessages(dedupe ? dedupeMessages(allMessages) : allMessages);
    const start = Math.max(0, Number.parseInt(cursor, 10) || 0);
    const cap = normalizeContextLimit(limit);
    const pagedMessages = cap === 0 ? orderedMessages.slice(start) : orderedMessages.slice(start, start + cap);
    const nextCursor = cap !== 0 && start + cap < orderedMessages.length ? start + cap : null;
    const result = {
      ok: true,
      source: 'ctm-db',
      db_path: handle.dbPath,
      active_source: handle.activeSource || 'ctm-db',
      requested_ids: identifiers,
      sessions,
      count: pagedMessages.length,
      total_count: orderedMessages.length,
      cursor: start,
      next_cursor: nextCursor,
      messages: pagedMessages,
      transfer: {
        full_context_supported: true,
        served_from: servedFromRows ? 'session_message_rows' : 'session_conversations/session_messages',
        jsonl_fallback_used: false,
        deduped: Boolean(dedupe),
      },
    };

    if (format === 'markdown' || format === 'compact') {
      result.text = renderContextMarkdown(result, { compact: format === 'compact' });
    }
    return result;
  });
}

function buildContextPack({
  task,
  query,
  session_ids,
  ids,
  limit = 5,
  token_budget = DEFAULT_TOKEN_BUDGET,
  include_raw = false,
  mode = 'auto',
  db,
  dbPath,
  env,
} = {}) {
  const identifiers = normalizeIdentifiers(session_ids || ids);
  const effectiveQuery = String(query || task || '').trim();
  const maxSessions = clampLimit(limit, 5, MAX_SEARCH_LIMIT);
  const budgetChars = Math.max(1000, Math.min(Number(token_budget || DEFAULT_TOKEN_BUDGET), 200000) * 4);

  return withCtmDb({ db, dbPath, env }, (conn, handle) => {
    const selectedIds = [];
    const search = effectiveQuery
      ? searchCtmSessions({ query: effectiveQuery, limit: maxSessions, db: conn, dbPath: handle.dbPath })
      : { ok: true, results: [] };

    for (const id of identifiers) selectedIds.push(id);
    for (const item of search.results || []) {
      if (item.session_id) selectedIds.push(item.session_id);
      if (item.agent_session_id) selectedIds.push(item.agent_session_id);
    }

    const uniqueIds = [...new Set(selectedIds)].slice(0, maxSessions);
    const context = getCtmSessionContext({
      session_ids: uniqueIds,
      limit: mode === 'full' ? 0 : 1000,
      include_raw,
      dedupe: true,
      db: conn,
      dbPath: handle.dbPath,
    });

    const pack = trimContextToBudget(context, budgetChars);
    return {
      ok: true,
      source: 'ctm-db',
      db_path: handle.dbPath,
      active_source: handle.activeSource || 'ctm-db',
      task: task || '',
      query: effectiveQuery,
      mode,
      search_results: search.results || [],
      selected_session_ids: uniqueIds,
      sessions: pack.sessions,
      messages: pack.messages,
      text: renderContextMarkdown(pack, { compact: mode !== 'full' }),
      transfer: {
        full_context_supported: true,
        served_from: 'session_conversations/session_messages',
        jsonl_fallback_used: false,
        deduped: true,
        token_budget: Number(token_budget || DEFAULT_TOKEN_BUDGET),
        truncated: pack.truncated,
      },
    };
  });
}

function hasAnySessionTable(db) {
  return inspectSessionTables(db).ok;
}

function inspectSessionTables(db) {
  try {
    const placeholders = CTM_SESSION_TABLES.map(() => '?').join(', ');
    const rows = db.prepare(
      `SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name IN (${placeholders})`
    ).all(...CTM_SESSION_TABLES);
    const tables = rows.map((row) => row.name).filter(Boolean).sort();
    const missingTables = CTM_SESSION_TABLES.filter((table) => !tables.includes(table));
    const tableCounts = Object.fromEntries(
      CTM_SESSION_TABLES
        .filter((table) => tables.includes(table))
        .map((table) => [table, tableRowCount(db, table)])
    );
    if (tables.length === 0) {
      return {
        ok: false,
        reason: 'ctm_session_tables_missing',
        tables,
        missing_tables: missingTables,
        table_counts: tableCounts,
      };
    }
    return { ok: true, reason: '', tables, missing_tables: missingTables, table_counts: tableCounts };
  } catch (err) {
    const classified = classifySqliteFailure(err, 'ctm_db_schema_read_failed');
    return {
      ok: false,
      reason: classified.reason,
      error: err?.message || String(err),
      tables: [],
      missing_tables: [...CTM_SESSION_TABLES],
    };
  }
}

function getCtmDbHealth(options = {}) {
  return withCtmDb(options, (conn, handle) => {
    const tableStatus = inspectSessionTables(conn);
    return {
      ok: tableStatus.ok,
      source: 'ctm-db',
      unavailable: false,
      db_path: handle.dbPath,
      ...tableStatus,
    };
  });
}

function hasTable(db, name) {
  try {
    return Boolean(db.prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name = ?").get(name));
  } catch {
    return false;
  }
}

function tableColumns(db, table) {
  try {
    return db.prepare(`PRAGMA table_info(${quoteIdent(table)})`).all().map((row) => row.name);
  } catch {
    return [];
  }
}

function idColumnFor(db, table) {
  const cols = tableColumns(db, table);
  if (cols.includes('ctm_session_id')) return 'ctm_session_id';
  if (cols.includes('session_id')) return 'session_id';
  return null;
}

function quoteIdent(value) {
  return `"${String(value).replace(/"/g, '""')}"`;
}

function searchSessionMessagesFts(db, query, limit) {
  const ftsQueries = buildFtsQueries(query);
  if (!ftsQueries.length) return [];
  // Phase: prefer the faithful per-message FTS (session_message_rows_fts); fall back to the legacy
  // session_messages_fts only while it still exists (sessions not yet backfilled into rows). Each
  // index is tried independently so a mid-retirement drop of the legacy table can't blank results.
  const tryIndex = (cfg) => {
    if (!hasTable(db, cfg.table) || !hasTable(db, cfg.fts)) return null;
    const idCol = cfg.idCol || idColumnFor(db, cfg.table);
    if (!idCol) return null;
    try {
      const stmt = db.prepare(`
        SELECT sm.${quoteIdent(idCol)} AS conversation_id, sm.id AS message_rowid, sm.role,
               sm.message_index, sm.${cfg.textCol} AS content, sm.${cfg.tsCol} AS created_at, fts.rank AS rank
        FROM ${cfg.fts} fts JOIN ${cfg.table} sm ON sm.id = fts.rowid
        WHERE ${cfg.fts} MATCH ? ORDER BY fts.rank LIMIT ?
      `);
      for (const ftsQuery of ftsQueries) {
        const rows = stmt.all(ftsQuery, limit);
        if (rows.length) return rows.map((row, index) => enrichSearchRow(db, row, { source_table: cfg.fts, rank: index, score: row.rank }));
      }
      return [];
    } catch { return null; }
  };
  const rowsRes = tryIndex({ table: 'session_message_rows', fts: 'session_message_rows_fts', idCol: 'ctm_session_id', textCol: 'text', tsCol: 'timestamp' });
  if (rowsRes && rowsRes.length) return rowsRes;
  const legacyRes = tryIndex({ table: 'session_messages', fts: 'session_messages_fts', textCol: 'content', tsCol: 'created_at' });
  return legacyRes || rowsRes || [];
}

function searchSessionMessagesLike(db, query, limit) {
  const like = `%${escapeLike(query)}%`;
  // Prefer the faithful rows (text column); fall back to the legacy session_messages (content)
  // only while it exists. Each is tried independently.
  const tryLike = (cfg) => {
    if (!hasTable(db, cfg.table)) return null;
    const idCol = cfg.idCol || idColumnFor(db, cfg.table);
    if (!idCol) return null;
    try {
      const rows = db.prepare(`
        SELECT ${quoteIdent(idCol)} AS conversation_id, id AS message_rowid, role, message_index,
               ${cfg.textCol} AS content, ${cfg.tsCol} AS created_at
        FROM ${cfg.table}
        WHERE ${cfg.textCol} LIKE ? ESCAPE '\\'
        ORDER BY ${cfg.tsCol} DESC, id DESC LIMIT ?
      `).all(like, limit);
      return rows.map((row, index) => enrichSearchRow(db, row, { source_table: cfg.table, rank: index + 1000 }));
    } catch { return null; }
  };
  const rowsRes = tryLike({ table: 'session_message_rows', idCol: 'ctm_session_id', textCol: 'text', tsCol: 'timestamp' });
  if (rowsRes && rowsRes.length) return rowsRes;
  const legacyRes = tryLike({ table: 'session_messages', textCol: 'content', tsCol: 'created_at' });
  return legacyRes || rowsRes || [];
}

function searchSessionConversationMessages(db, query, limit) {
  if (!hasTable(db, 'session_conversations')) return [];
  const convIdCol = idColumnFor(db, 'session_conversations');
  if (!convIdCol) return [];
  const cols = tableColumns(db, 'session_conversations');
  if (!cols.includes('messages')) return [];
  const terms = searchTerms(query);
  const predicates = terms.length
    ? terms.slice(0, 6).map(() => `messages LIKE ? ESCAPE '\\'`)
    : [`messages LIKE ? ESCAPE '\\'`];
  const params = terms.length
    ? terms.slice(0, 6).map((term) => `%${escapeLike(term)}%`)
    : [`%${escapeLike(query)}%`];
  const selectCols = [
    `${quoteIdent(convIdCol)} AS _conversation_id`,
    'messages',
    cols.includes('project_path') ? 'project_path' : "'' AS project_path",
    cols.includes('title') ? 'title' : "'' AS title",
    cols.includes('first_message') ? 'first_message' : "'' AS first_message",
    cols.includes('git_branch') ? 'git_branch' : "'' AS git_branch",
    cols.includes('session_created_at') ? 'session_created_at' : "'' AS session_created_at",
    cols.includes('imported_at') ? 'imported_at' : "'' AS imported_at",
    cols.includes('model_provider') ? 'model_provider' : "'' AS model_provider",
    cols.includes('model_id') ? 'model_id' : "'' AS model_id",
  ];
  try {
    const rows = db.prepare(`
      SELECT ${selectCols.join(', ')}
      FROM session_conversations
      WHERE ${predicates.join(' OR ')}
      ORDER BY imported_at DESC
      LIMIT ?
    `).all(...params, Math.max(limit, 20));
    const results = [];
    for (const row of rows) {
      const conversationId = row._conversation_id;
      const bundle = resolveSessionBundle(db, conversationId) || {
        ctm_session_id: resolveConversationOwner(db, conversationId).ctm_session_id || conversationId,
        agent_session_ids: [],
        provider: row.model_provider || '',
        cwd: row.project_path || '',
        project_path: row.project_path || '',
        title: row.title || row.first_message || '',
      };
      // Candidates are found via the blob LIKE (still dual-written); reconstruct the messages
      // from the faithful rows when present so search reads the same source as getCtmSessionContext.
      const messages = readSessionMessageRows(db, conversationId) || parseConversationMessages(row);
      for (let i = 0; i < messages.length; i += 1) {
        const normalized = normalizeMessage(messages[i], {
          row,
          bundle,
          index: i,
          includeRaw: false,
        });
        if (!normalized.content || !matchesQueryText(normalized.content, query, terms)) continue;
        results.push({
          ...normalized,
          source_table: 'session_conversations.messages',
          snippet: snippetAround(normalized.content, 800),
          rank: 500 + results.length,
        });
        if (results.length >= limit) return results;
      }
    }
    return results;
  } catch {
    return [];
  }
}

function searchSessionMetadata(db, query, limit) {
  const results = [];
  const like = `%${escapeLike(query)}%`;
  if (hasTable(db, 'session_conversations')) {
    const convIdCol = idColumnFor(db, 'session_conversations');
    if (convIdCol) {
      const cols = tableColumns(db, 'session_conversations');
      const selectCols = [
        `${quoteIdent(convIdCol)} AS conversation_id`,
        cols.includes('project_path') ? 'project_path' : "'' AS project_path",
        cols.includes('title') ? 'title' : "'' AS title",
        cols.includes('first_message') ? 'first_message' : "'' AS first_message",
        cols.includes('git_branch') ? 'git_branch' : "'' AS git_branch",
        cols.includes('imported_at') ? 'imported_at' : "'' AS imported_at",
      ];
      const predicates = ['1=0'];
      const params = [];
      for (const col of ['title', 'first_message', 'project_path']) {
        if (cols.includes(col)) {
          predicates.push(`${quoteIdent(col)} LIKE ? ESCAPE '\\'`);
          params.push(like);
        }
      }
      try {
        const rows = db.prepare(`
          SELECT ${selectCols.join(', ')}
          FROM session_conversations
          WHERE ${predicates.join(' OR ')}
          ORDER BY imported_at DESC
          LIMIT ?
        `).all(...params, limit);
        results.push(...rows.map((row, index) => enrichMetadataRow(db, row, index)));
      } catch {}
    }
  }

  if (hasTable(db, 'ctm_sessions')) {
    try {
      const rows = db.prepare(`
        SELECT id AS ctm_session_id, provider, project_path, cwd, title, updated_at
        FROM ctm_sessions
        WHERE id = ? OR title LIKE ? ESCAPE '\\' OR project_path LIKE ? ESCAPE '\\' OR cwd LIKE ? ESCAPE '\\'
        ORDER BY updated_at DESC
        LIMIT ?
      `).all(query, like, like, like, limit);
      results.push(...rows.map((row, index) => ({
        source: 'ctm-db',
        source_table: 'ctm_sessions',
        session_id: row.ctm_session_id,
        ctm_session_id: row.ctm_session_id,
        provider: row.provider || '',
        cwd: row.cwd || row.project_path || '',
        title: row.title || '',
        timestamp: row.updated_at || '',
        snippet: row.title || row.cwd || row.project_path || row.ctm_session_id,
        rank: index + 2000,
      })));
    } catch {}
  }

  if (hasTable(db, 'agent_sessions')) {
    try {
      const rows = db.prepare(`
        SELECT agent_session_id, ctm_session_id, provider, project_path, first_message, modified_at, model, git_branch
        FROM agent_sessions
        WHERE agent_session_id = ? OR ctm_session_id = ? OR first_message LIKE ? ESCAPE '\\' OR project_path LIKE ? ESCAPE '\\'
        ORDER BY modified_at DESC, updated_at DESC
        LIMIT ?
      `).all(query, query, like, like, limit);
      results.push(...rows.map((row, index) => ({
        source: 'ctm-db',
        source_table: 'agent_sessions',
        session_id: row.ctm_session_id || row.agent_session_id,
        ctm_session_id: row.ctm_session_id || '',
        agent_session_id: row.agent_session_id || '',
        provider: row.provider || '',
        cwd: row.project_path || '',
        branch: row.git_branch || '',
        model: row.model || '',
        timestamp: row.modified_at || '',
        snippet: row.first_message || row.project_path || row.agent_session_id,
        rank: index + 3000,
      })));
    } catch {}
  }

  return results;
}

function searchTerms(query) {
  return String(query || '')
    .match(/[A-Za-z0-9_./:-]+/g)
    ?.map((term) => term.replace(/^[-:.\\/]+|[-:.\\/]+$/g, '').toLowerCase())
    .filter((term) => term.length > 1)
    .slice(0, 12) || [];
}

function matchesQueryText(content, query, terms = searchTerms(query)) {
  const normalized = normalizeContent(content);
  const phrase = normalizeContent(query);
  if (phrase && normalized.includes(phrase)) return true;
  if (!terms.length) return false;
  return terms.every((term) => normalized.includes(term.toLowerCase()));
}

function enrichSearchRow(db, row, extra = {}) {
  const resolved = resolveConversationOwner(db, row.conversation_id);
  return {
    source: 'ctm-db',
    source_table: extra.source_table,
    session_id: resolved.ctm_session_id || row.conversation_id,
    ctm_session_id: resolved.ctm_session_id || '',
    agent_session_id: resolved.agent_session_id || '',
    conversation_id: row.conversation_id,
    role: row.role || '',
    message_index: row.message_index,
    timestamp: row.created_at || resolved.modified_at || resolved.updated_at || '',
    cwd: resolved.cwd || resolved.project_path || '',
    branch: resolved.git_branch || '',
    title: resolved.title || resolved.first_message || '',
    provider: resolved.provider || '',
    model: resolved.model || '',
    snippet: snippetAround(row.content || '', 800),
    content_hash: contentFingerprint(row.content || ''),
    rank: extra.rank || 0,
  };
}

function enrichMetadataRow(db, row, rank) {
  const resolved = resolveConversationOwner(db, row.conversation_id);
  return {
    source: 'ctm-db',
    source_table: 'session_conversations',
    session_id: resolved.ctm_session_id || row.conversation_id,
    ctm_session_id: resolved.ctm_session_id || '',
    agent_session_id: resolved.agent_session_id || '',
    conversation_id: row.conversation_id,
    timestamp: row.imported_at || resolved.modified_at || '',
    cwd: resolved.cwd || row.project_path || '',
    branch: resolved.git_branch || row.git_branch || '',
    title: resolved.title || row.title || row.first_message || '',
    provider: resolved.provider || '',
    model: resolved.model || '',
    snippet: row.title || row.first_message || row.project_path || row.conversation_id,
    content_hash: contentFingerprint([row.title, row.first_message, row.project_path].filter(Boolean).join('\n')),
    rank: rank + 1500,
  };
}

function resolveConversationOwner(db, conversationId) {
  const id = String(conversationId || '');
  const empty = { ctm_session_id: id, agent_session_id: '', provider: '', project_path: '', cwd: '', title: '' };
  if (!id) return empty;

  let agent = null;
  if (hasTable(db, 'agent_sessions')) {
    try {
      agent = db.prepare('SELECT * FROM agent_sessions WHERE agent_session_id = ?').get(id) || null;
    } catch {}
  }
  const ctmId = agent?.ctm_session_id || id;
  let ctm = null;
  if (hasTable(db, 'ctm_sessions')) {
    try {
      ctm = db.prepare('SELECT * FROM ctm_sessions WHERE id = ?').get(ctmId) || null;
    } catch {}
  }
  if (!agent && hasTable(db, 'agent_sessions')) {
    try {
      agent = db.prepare('SELECT * FROM agent_sessions WHERE ctm_session_id = ? ORDER BY modified_at DESC, created_at DESC LIMIT 1').get(ctmId) || null;
    } catch {}
  }
  return {
    ctm_session_id: ctm?.id || agent?.ctm_session_id || ctmId,
    agent_session_id: agent?.agent_session_id || (agent?.ctm_session_id ? id : ''),
    provider: ctm?.provider || agent?.provider || '',
    project_path: ctm?.project_path || agent?.project_path || '',
    cwd: ctm?.cwd || ctm?.project_path || agent?.project_path || '',
    title: ctm?.title || agent?.first_message || '',
    first_message: agent?.first_message || '',
    git_branch: agent?.git_branch || '',
    model: agent?.model || '',
    modified_at: agent?.modified_at || '',
    updated_at: ctm?.updated_at || '',
  };
}

function resolveSessionBundle(db, rawId) {
  const candidates = candidateIds(rawId);
  for (const candidate of candidates) {
    const bundle = resolveSessionBundleExact(db, candidate, rawId);
    if (bundle) return bundle;
  }
  return null;
}

function resolveSessionBundleExact(db, id, requestedId = id) {
  const requested = String(requestedId || id || '').trim();
  const cleanId = String(id || '').trim();
  if (!cleanId) return null;

  let ctm = null;
  let agent = null;
  if (hasTable(db, 'ctm_sessions')) {
    try { ctm = db.prepare('SELECT * FROM ctm_sessions WHERE id = ?').get(cleanId) || null; } catch {}
  }
  if (hasTable(db, 'agent_sessions')) {
    try { agent = db.prepare('SELECT * FROM agent_sessions WHERE agent_session_id = ?').get(cleanId) || null; } catch {}
  }

  const ctmId = ctm?.id || agent?.ctm_session_id || cleanId;
  if (!ctm && hasTable(db, 'ctm_sessions')) {
    try { ctm = db.prepare('SELECT * FROM ctm_sessions WHERE id = ?').get(ctmId) || null; } catch {}
  }

  let agents = [];
  if (hasTable(db, 'agent_sessions')) {
    try {
      agents = db.prepare('SELECT * FROM agent_sessions WHERE ctm_session_id = ? ORDER BY modified_at DESC, created_at DESC').all(ctmId);
    } catch {}
  }
  if (agent && !agents.some((row) => row.agent_session_id === agent.agent_session_id)) agents.unshift(agent);

  const conversationIds = new Set();
  for (const idCandidate of [cleanId, ctmId, ...agents.map((row) => row.agent_session_id)]) {
    if (idCandidate && hasConversation(db, idCandidate)) conversationIds.add(idCandidate);
  }

  if (!ctm && !agent && conversationIds.size === 0) return null;
  return {
    requested_id: requested,
    ctm_session_id: ctmId,
    agent_session_ids: agents.map((row) => row.agent_session_id).filter(Boolean),
    conversation_ids: [...conversationIds],
    provider: ctm?.provider || agent?.provider || '',
    project_path: ctm?.project_path || agent?.project_path || '',
    cwd: ctm?.cwd || ctm?.project_path || agent?.project_path || '',
    title: ctm?.title || agent?.first_message || '',
    starred: Boolean(ctm?.starred),
    user_renamed: Boolean(ctm?.user_renamed),
    created_at: ctm?.created_at || agent?.created_at || '',
    updated_at: ctm?.updated_at || agent?.updated_at || agent?.modified_at || '',
    agents,
  };
}

function hasConversation(db, sessionId) {
  if (!hasTable(db, 'session_conversations')) return false;
  const idCol = idColumnFor(db, 'session_conversations');
  if (!idCol) return false;
  try {
    return Boolean(db.prepare(`SELECT 1 FROM session_conversations WHERE ${quoteIdent(idCol)} = ? LIMIT 1`).get(sessionId));
  } catch {
    return false;
  }
}

function getConversationRows(db, bundle) {
  if (!hasTable(db, 'session_conversations')) return [];
  const idCol = idColumnFor(db, 'session_conversations');
  if (!idCol) return [];
  const ids = bundle.conversation_ids && bundle.conversation_ids.length
    ? bundle.conversation_ids
    : [bundle.ctm_session_id, ...bundle.agent_session_ids].filter(Boolean);
  const rows = [];
  const seen = new Set();
  for (const id of ids) {
    if (seen.has(id)) continue;
    seen.add(id);
    try {
      const row = db.prepare(`SELECT *, ${quoteIdent(idCol)} AS _conversation_id FROM session_conversations WHERE ${quoteIdent(idCol)} = ?`).get(id);
      if (row) rows.push(row);
    } catch {}
  }
  return rows.sort((a, b) => compareTimestamp(a.session_created_at || a.imported_at, b.session_created_at || b.imported_at));
}

function parseConversationMessages(row) {
  try {
    const parsed = JSON.parse(row?.messages || '[]');
    return Array.isArray(parsed) ? parsed : [];
  } catch {
    return [];
  }
}

// Phase 5: read the faithful per-message rows (session_message_rows) instead of the blob.
// The rows ARE the same parsed message objects the blob held (CTM writes them in lockstep),
// so this returns a byte-identical array — no re-embedding, no parser. Returns null when the
// table/rows are absent (pre-migration or a half-migrated session) so the caller falls back
// to parseConversationMessages(row) (the blob). Keyed by the SAME id the blob row uses.
function readSessionMessageRows(db, conversationId) {
  if (!conversationId || !hasTable(db, 'session_message_rows')) return null;
  try {
    const rows = db.prepare(
      'SELECT role, text, timestamp, meta FROM session_message_rows WHERE ctm_session_id = ? ORDER BY message_index ASC'
    ).all(conversationId);
    if (!rows.length) return null;
    return rows.map(messageRowFromColumns);
  } catch {
    return null;
  }
}

// Rebuild the original importer message object: non-standard messages are stored VERBATIM in
// `meta`; simple ones are pure columns ({role,text,timestamp}). Mirrors db.js _messageRowToObject.
function messageRowFromColumns(r) {
  if (r && r.meta) {
    try {
      const obj = JSON.parse(r.meta);
      if (obj && typeof obj === 'object') return obj;
    } catch { /* fall through to columns */ }
  }
  return { role: r.role, text: r.text, timestamp: r.timestamp || '' };
}

function normalizeMessage(raw, { row, bundle, index, includeRaw }) {
  const role = normalizeRole(raw);
  const content = messageText(raw);
  const conversationId = row?._conversation_id || row?.ctm_session_id || row?.session_id || bundle.ctm_session_id;
  return {
    source: 'ctm-db',
    source_table: 'session_conversations',
    session_id: bundle.ctm_session_id,
    ctm_session_id: bundle.ctm_session_id,
    agent_session_id: conversationId !== bundle.ctm_session_id ? conversationId : '',
    conversation_id: conversationId,
    message_index: Number.isInteger(raw?.message_index) ? raw.message_index : index,
    role,
    timestamp: raw?.timestamp || raw?.created_at || row?.session_created_at || row?.imported_at || '',
    content,
    content_hash: contentFingerprint(`${role}\n${content}`),
    metadata: compactObject({
      provider: bundle.provider || row?.model_provider || '',
      model: row?.model_id || '',
      title: bundle.title || row?.title || '',
      cwd: bundle.cwd || row?.project_path || '',
      branch: row?.git_branch || '',
      tool_calls: raw?.tool_calls || raw?.toolCalls,
      files: raw?.files || raw?.filesEdited,
    }),
    raw: includeRaw ? raw : undefined,
  };
}

function normalizeRole(raw) {
  const role = raw?.role || raw?.speaker || raw?.type || '';
  if (role === 'human') return 'user';
  if (role === 'ai') return 'assistant';
  if (role === 'tool_result') return 'tool';
  return String(role || 'unknown').toLowerCase();
}

function messageText(raw) {
  if (!raw) return '';
  if (typeof raw.content === 'string') return raw.content.trim();
  if (Array.isArray(raw.content)) {
    return raw.content.map((part) => {
      if (!part) return '';
      if (typeof part === 'string') return part;
      if (typeof part.text === 'string') return part.text;
      if (typeof part.content === 'string') return part.content;
      if (typeof part.input === 'object') return JSON.stringify(part.input);
      return '';
    }).filter(Boolean).join('\n').trim();
  }
  if (typeof raw.text === 'string') return raw.text.trim();
  if (typeof raw.message === 'string') return raw.message.trim();
  if (raw.message && typeof raw.message.content === 'string') return raw.message.content.trim();
  return '';
}

function formatSessionSummary(bundle, conversations, messages) {
  const firstConversation = conversations[0] || {};
  return {
    session_id: bundle.ctm_session_id,
    ctm_session_id: bundle.ctm_session_id,
    agent_session_ids: bundle.agent_session_ids || [],
    conversation_ids: conversations.map((row) => row._conversation_id || row.ctm_session_id || row.session_id).filter(Boolean),
    provider: bundle.provider || firstConversation.model_provider || '',
    cwd: bundle.cwd || firstConversation.project_path || '',
    branch: firstConversation.git_branch || bundle.agents?.find((a) => a.git_branch)?.git_branch || '',
    title: bundle.title || firstConversation.title || firstConversation.first_message || '',
    created_at: bundle.created_at || firstConversation.session_created_at || '',
    updated_at: bundle.updated_at || firstConversation.imported_at || '',
    message_count: messages.length,
    user_msg_count: firstConversation.user_msg_count || messages.filter((msg) => msg.role === 'user').length,
    assistant_msg_count: firstConversation.assistant_msg_count || messages.filter((msg) => msg.role === 'assistant').length,
  };
}

function orderMessages(messages) {
  return [...messages].sort((a, b) => {
    const t = compareTimestamp(a.timestamp, b.timestamp);
    if (t !== 0) return t;
    return (a.message_index || 0) - (b.message_index || 0);
  });
}

function compareTimestamp(a, b) {
  const at = Date.parse(a || '');
  const bt = Date.parse(b || '');
  if (Number.isFinite(at) && Number.isFinite(bt) && at !== bt) return at - bt;
  if (Number.isFinite(at) && !Number.isFinite(bt)) return -1;
  if (!Number.isFinite(at) && Number.isFinite(bt)) return 1;
  return 0;
}

function dedupeMessages(messages) {
  return dedupeItems(messages, {
    contentFields: ['content'],
    keyPrefix: (item) => item.role || '',
    minContentLength: 12,
  });
}

function dedupeItems(items, {
  limit = 0,
  contentFields = ['content'],
  keyPrefix = () => '',
  minContentLength = 0,
} = {}) {
  const seen = new Map();
  for (const item of items || []) {
    const content = contentFields.map((field) => item?.[field]).find((value) => typeof value === 'string' && value.trim()) || '';
    const normalized = normalizeContent(content);
    const prefix = typeof keyPrefix === 'function' ? keyPrefix(item) : String(keyPrefix || '');
    const key = normalized.length >= minContentLength
      ? `${prefix}:${contentFingerprint(normalized)}`
      : `${prefix}:${item.session_id || ''}:${item.conversation_id || ''}:${item.message_index ?? item.id ?? normalized}`;

    if (!seen.has(key)) {
      seen.set(key, { ...item });
      continue;
    }
    const existing = seen.get(key);
    seen.set(key, mergeDuplicateItem(existing, item));
  }
  const deduped = [...seen.values()].sort((a, b) => (a.rank || 0) - (b.rank || 0));
  return limit ? deduped.slice(0, limit) : deduped;
}

function mergeDuplicateItem(existing, next) {
  const merged = { ...existing };
  const provenance = [
    ...(Array.isArray(existing.provenance) ? existing.provenance : [provenanceOf(existing)]),
    provenanceOf(next),
  ].filter(Boolean);
  merged.provenance = dedupeProvenance(provenance);
  if (!merged.agent_session_id && next.agent_session_id) merged.agent_session_id = next.agent_session_id;
  if (!merged.ctm_session_id && next.ctm_session_id) merged.ctm_session_id = next.ctm_session_id;
  if (!merged.conversation_id && next.conversation_id) merged.conversation_id = next.conversation_id;
  if (!merged.timestamp && next.timestamp) merged.timestamp = next.timestamp;
  if ((String(next.snippet || '').length > String(merged.snippet || '').length) && String(next.snippet || '').length <= 1200) {
    merged.snippet = next.snippet;
  }
  return merged;
}

function provenanceOf(item) {
  if (!item) return null;
  return compactObject({
    source: item.source,
    source_table: item.source_table,
    session_id: item.session_id,
    ctm_session_id: item.ctm_session_id,
    agent_session_id: item.agent_session_id,
    conversation_id: item.conversation_id,
    message_index: item.message_index,
    id: item.id,
  });
}

function dedupeProvenance(items) {
  const seen = new Set();
  const out = [];
  for (const item of items) {
    const key = JSON.stringify(item);
    if (seen.has(key)) continue;
    seen.add(key);
    out.push(item);
  }
  return out;
}

function normalizeContent(value) {
  return String(value || '')
    .replace(/\r\n/g, '\n')
    .replace(/\s+/g, ' ')
    .trim()
    .toLowerCase();
}

function contentFingerprint(value) {
  return crypto.createHash('sha256').update(String(value || '')).digest('hex').slice(0, 16);
}

function buildFtsQueries(query) {
  const terms = String(query || '')
    .match(/[A-Za-z0-9_./:-]+/g)
    ?.map((term) => term.replace(/^[-:.\\/]+|[-:.\\/]+$/g, ''))
    .filter((term) => term.length > 1)
    .slice(0, 12) || [];
  if (!terms.length) return [];
  const quoted = terms.map((term) => `"${term.replace(/"/g, '""')}"`);
  const andQuery = quoted.join(' AND ');
  const orQuery = quoted.join(' OR ');
  return andQuery === orQuery ? [andQuery] : [andQuery, orQuery];
}

function escapeLike(value) {
  return String(value || '').replace(/[\\%_]/g, (match) => `\\${match}`);
}

function snippetAround(content, max = 800) {
  const text = String(content || '').replace(/\s+/g, ' ').trim();
  if (text.length <= max) return text;
  return `${text.slice(0, Math.max(0, max - 1)).trim()}...`;
}

function normalizeIdentifiers(value) {
  if (Array.isArray(value)) return value.flatMap((item) => normalizeIdentifiers(item));
  if (value == null) return [];
  return String(value).split(',').map((part) => part.trim()).filter(Boolean);
}

function candidateIds(id) {
  const raw = String(id || '').trim();
  if (!raw) return [];
  const out = [raw];
  const parts = raw.split(':').filter(Boolean);
  if (parts.length > 1) {
    out.push(parts[parts.length - 1]);
    if (parts.length >= 4 && parts[0] === 'diary') out.push(parts.slice(2, -1).join(':'));
    if (parts.length >= 2) out.push(parts.slice(1).join(':'));
  }
  return [...new Set(out.filter(Boolean))];
}

function clampLimit(value, fallback, max) {
  const n = Number(value || fallback);
  if (!Number.isFinite(n)) return fallback;
  return Math.min(Math.max(Math.trunc(n), 1), max);
}

function normalizeContextLimit(value) {
  if (value === 0 || value === '0' || value === 'all' || value === 'full') return 0;
  return clampLimit(value, DEFAULT_CONTEXT_LIMIT, MAX_CONTEXT_LIMIT);
}

function trimContextToBudget(context, budgetChars) {
  const sessions = context.sessions || [];
  const messages = [];
  let used = JSON.stringify(sessions).length;
  let truncated = false;
  for (const message of context.messages || []) {
    const size = String(message.content || '').length + 200;
    if (used + size > budgetChars) {
      truncated = true;
      break;
    }
    messages.push(message);
    used += size;
  }
  return {
    ...context,
    sessions,
    messages,
    count: messages.length,
    truncated,
  };
}

function renderContextMarkdown(context, { compact = false } = {}) {
  const lines = [];
  lines.push('# Wall-E Session Context');
  lines.push('');
  if (context.task) {
    lines.push(`Task: ${context.task}`);
    lines.push('');
  }
  if (context.sessions?.length) {
    lines.push('## Sessions');
    for (const session of context.sessions) {
      const title = session.title ? ` - ${session.title}` : '';
      const cwd = session.cwd ? ` (${session.cwd})` : '';
      lines.push(`- ${session.session_id}${title}${cwd}; messages=${session.message_count}`);
    }
    lines.push('');
  }
  lines.push('## Messages');
  const maxContent = compact ? 1200 : 8000;
  for (const msg of context.messages || []) {
    const label = [msg.role || 'unknown', msg.timestamp || '', msg.session_id || ''].filter(Boolean).join(' | ');
    lines.push(`### ${label}`);
    const content = String(msg.content || '');
    lines.push(content.length > maxContent ? `${content.slice(0, maxContent).trim()}...` : content);
    lines.push('');
  }
  if (context.transfer?.truncated || context.truncated) {
    lines.push('_Context pack truncated to fit the requested budget._');
  }
  return lines.join('\n').trim();
}

function ensureMessageIndexTables(db) {
  const created = [];
  if (!hasTable(db, 'session_messages')) {
    db.exec(`
      CREATE TABLE session_messages (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        ctm_session_id TEXT NOT NULL,
        message_index INTEGER NOT NULL,
        role TEXT NOT NULL,
        content TEXT NOT NULL,
        created_at TEXT,
        UNIQUE(ctm_session_id, message_index)
      )
    `);
    created.push('session_messages');
  }
  if (!hasTable(db, 'session_messages_fts')) {
    db.exec(`
      CREATE VIRTUAL TABLE session_messages_fts USING fts5(
        content,
        content='', contentless_delete=1, tokenize='porter unicode61'
      )
    `);
    created.push('session_messages_fts');
  }
  return created;
}

function backfillCtmSessionMessages({ db, dbPath, limit = 100, dry_run = false, allow_write = false } = {}) {
  if (!dry_run && !allow_write) {
    return {
      ok: false,
      source: 'ctm-db',
      reason: 'ctm_db_write_requires_ctm_api',
      dry_run: false,
    };
  }
  return withCtmDb({ db, dbPath, readonly: Boolean(dry_run), fallback: false }, (conn, handle) => {
    const tableStatus = inspectSessionTables(conn);
    if (!tableStatus.ok && tableStatus.reason !== 'ctm_session_tables_missing') {
      return {
        ok: false,
        source: 'ctm-db',
        db_path: handle.dbPath,
        active_source: handle.activeSource || 'ctm-db',
        ...tableStatus,
      };
    }
    if (!hasTable(conn, 'session_conversations')) {
      return {
        ok: false,
        source: 'ctm-db',
        db_path: handle.dbPath,
        active_source: handle.activeSource || 'ctm-db',
        reason: 'session_conversations_missing',
      };
    }
    if (!idColumnFor(conn, 'session_conversations')) {
      return {
        ok: false,
        source: 'ctm-db',
        db_path: handle.dbPath,
        active_source: handle.activeSource || 'ctm-db',
        reason: 'session_conversations_id_missing',
      };
    }

    const created_tables = dry_run ? [] : ensureMessageIndexTables(conn);
    if (dry_run && (!hasTable(conn, 'session_messages') || !hasTable(conn, 'session_messages_fts'))) {
      return backfillPlan(conn, { limit, created_tables: ['session_messages', 'session_messages_fts'] });
    }

    const rows = selectBackfillConversationRows(conn, { limit });
    const plan = backfillPlanFromRows(rows, { created_tables });
    if (dry_run) return plan;

    const msgSessionCol = idColumnFor(conn, 'session_messages') || 'ctm_session_id';
    const insertMsg = conn.prepare(`
      INSERT OR IGNORE INTO session_messages (${quoteIdent(msgSessionCol)}, message_index, role, content, created_at)
      VALUES (?, ?, ?, ?, ?)
    `);
    const insertFts = conn.prepare('INSERT INTO session_messages_fts(rowid, content) VALUES (?, ?)');
    let inserted_messages = 0;
    let inserted_fts = 0;
    const txn = conn.transaction((rows) => {
      for (const row of rows) {
        // Phase 5: build the index from the faithful rows when present (survives blob retirement).
        const messages = readSessionMessageRows(conn, row._conversation_id) || parseConversationMessages(row);
        let sessionInserted = 0;
        for (let i = 0; i < messages.length; i += 1) {
          const raw = messages[i];
          const role = normalizeRole(raw);
          if (role !== 'user' && role !== 'assistant') continue;
          const content = messageText(raw);
          if (content.length < 5) continue;
          const info = insertMsg.run(row._conversation_id, i, role, content, raw?.timestamp || raw?.created_at || null);
          if (info.changes > 0) {
            inserted_messages += 1;
            sessionInserted += 1;
            try {
              insertFts.run(info.lastInsertRowid, content.length > 2000 ? content.slice(0, 2000) : content);
              inserted_fts += 1;
            } catch {}
          }
        }
        if (sessionInserted === 0) {
          insertMsg.run(row._conversation_id, -1, 'system', '(no extractable messages)', null);
        }
      }
    });
    txn(rows);

    return {
      ...plan,
      dry_run: false,
      inserted_messages,
      inserted_fts,
      table_counts: inspectSessionTables(conn).table_counts || {},
    };
  });
}

function backfillPlan(db, { limit, created_tables = [] } = {}) {
  const rows = selectBackfillConversationRows(db, { limit });
  return backfillPlanFromRows(rows, { created_tables });
}

function selectBackfillConversationRows(db, { limit } = {}) {
  const convIdCol = idColumnFor(db, 'session_conversations');
  if (!convIdCol) {
    return [];
  }
  const cap = clampLimit(limit, 100, 1000);
  const hasMessages = hasTable(db, 'session_messages');
  const msgSessionCol = hasMessages ? idColumnFor(db, 'session_messages') : null;
  const join = hasMessages && msgSessionCol
    ? `LEFT JOIN session_messages sm ON sm.${quoteIdent(msgSessionCol)} = sc.${quoteIdent(convIdCol)}
       WHERE sm.${quoteIdent(msgSessionCol)} IS NULL`
    : '';
  return db.prepare(`
    SELECT sc.*, sc.${quoteIdent(convIdCol)} AS _conversation_id
    FROM session_conversations sc
    ${join}
    ORDER BY COALESCE(sc.imported_at, sc.session_created_at, '') DESC
    LIMIT ?
  `).all(cap);
}

function backfillPlanFromRows(rows, { created_tables = [] } = {}) {
  return {
    ok: true,
    source: 'ctm-db',
    dry_run: true,
    created_tables,
    planned_sessions: rows.length,
    sessions: rows.map((row) => ({
      conversation_id: row._conversation_id,
      project_path: row.project_path || '',
      title: row.title || row.first_message || '',
      user_msg_count: row.user_msg_count || 0,
      assistant_msg_count: row.assistant_msg_count || 0,
      imported_at: row.imported_at || '',
    })),
  };
}

function compactObject(obj) {
  const out = {};
  for (const [key, value] of Object.entries(obj || {})) {
    if (value === undefined || value === null || value === '') continue;
    if (Array.isArray(value) && value.length === 0) continue;
    out[key] = value;
  }
  return out;
}

module.exports = {
  CTM_SESSION_TABLES,
  resolveCtmDbPath,
  resolveCtmDbCandidatePaths,
  openCtmDb,
  hasOpenableCtmDb,
  inspectSessionTables,
  getCtmDbHealth,
  searchCtmSessions,
  getCtmSessionContext,
  buildContextPack,
  backfillCtmSessionMessages,
  dedupeItems,
  dedupeMessages,
  normalizeContent,
  contentFingerprint,
  renderContextMarkdown,
};
