// --- Prompt Harvester & Copilot Engine ---
// Extracts prompts from Claude Code session history, builds autocomplete index,
// manages prompt-session mapping, and powers the Prompt Copilot.

const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const dbModule = require('./db');
const sessionUtils = require('./session-utils');

// BM25 scoring from Wall-E (graceful fallback if unavailable)
let _bm25;
try { _bm25 = require('../wall-e/search/bm25'); } catch { _bm25 = null; }

// ============================================================
// Session Harvester — extracts user messages + assistant context
// ============================================================

function getHarvestState() {
  try {
    return dbModule.getDb().prepare('SELECT * FROM harvest_state WHERE id = 1').get() || null;
  } catch { return null; }
}

const HARVEST_STATE_COLS = new Set(['last_scan_at', 'files_scanned', 'prompts_harvested', 'last_lifecycle_refresh_at']);

function updateHarvestState(updates) {
  const d = dbModule.getDb();
  const safeUpdates = {};
  for (const [k, v] of Object.entries(updates)) {
    if (!HARVEST_STATE_COLS.has(k)) throw new Error(`Invalid harvest_state column: ${k}`);
    safeUpdates[k] = v;
  }
  const existing = d.prepare('SELECT id FROM harvest_state WHERE id = 1').get();
  if (existing) {
    const sets = Object.entries(safeUpdates).map(([k]) => `${k} = ?`).join(', ');
    d.prepare(`UPDATE harvest_state SET ${sets} WHERE id = 1`).run(...Object.values(safeUpdates));
  } else {
    d.prepare('INSERT INTO harvest_state (id, last_scan_at, files_scanned, prompts_harvested) VALUES (1, ?, ?, ?)').run(
      updates.last_scan_at || null, updates.files_scanned || 0, updates.prompts_harvested || 0
    );
  }
}

// Parse a single session JSONL file and extract all messages
function parseFullSession(filePath) {
  let content;
  try { content = fs.readFileSync(filePath, 'utf8'); }
  catch { return null; }

  const lines = content.split('\n').filter(Boolean);
  const messages = [];
  let sessionId = path.basename(filePath).replace(/\.jsonl(\.bak)?$/, '');
  let projectPath = '';
  let cwd = '';
  let version = '';

  for (const line of lines) {
    try {
      const entry = JSON.parse(line);
      if (entry.type === 'user' && entry.message?.role === 'user') {
        if (!projectPath && entry.cwd) {
          cwd = entry.cwd;
          version = entry.version || '';
        }

        const contentBlocks = entry.message.content;
        let text = '';
        const imageBlocks = [];

        if (typeof contentBlocks === 'string') {
          text = contentBlocks;
        } else if (Array.isArray(contentBlocks)) {
          for (const block of contentBlocks) {
            if (block.type === 'text') text += (text ? '\n' : '') + block.text;
            else if (block.type === 'image') imageBlocks.push(block);
          }
        }

        if (text.trim()) {
          messages.push({
            role: 'user',
            text: text.trim(),
            images: imageBlocks,
            timestamp: entry.timestamp,
            index: messages.length,
            uuid: entry.uuid,
          });
        }
      } else if (entry.type === 'assistant' && entry.message?.role === 'assistant') {
        const contentBlocks = entry.message.content;
        let text = '';
        const toolUses = [];

        if (typeof contentBlocks === 'string') {
          text = contentBlocks;
        } else if (Array.isArray(contentBlocks)) {
          for (const block of contentBlocks) {
            if (block.type === 'text') text += (text ? '\n' : '') + block.text;
            else if (block.type === 'tool_use') {
              toolUses.push({ name: block.name, input_summary: summarizeToolInput(block.name, block.input) });
            }
          }
        }

        if (text.trim() || toolUses.length) {
          messages.push({
            role: 'assistant',
            text: text.trim().slice(0, 2000), // Limit assistant text
            toolUses,
            index: messages.length,
            uuid: entry.uuid,
          });
        }
      }
    } catch { /* skip malformed lines */ }
  }

  return { sessionId, cwd, version, messages };
}

// Summarize tool input for context (keep it brief for autocomplete)
function summarizeToolInput(toolName, input) {
  if (!input) return '';
  switch (toolName) {
    case 'Read': return input.file_path || '';
    case 'Edit': return `${input.file_path || ''} (edit)`;
    case 'Write': return `${input.file_path || ''} (write)`;
    case 'Bash': return (input.command || '').slice(0, 100);
    case 'Glob': return input.pattern || '';
    case 'Grep': return input.pattern || '';
    default: return JSON.stringify(input).slice(0, 80);
  }
}

// Save a harvested image (base64 from session) — dedup via MD5 hash
function saveHarvestedImage(imageBlock, sessionId) {
  if (!imageBlock.source?.data) return null;
  try {
    const buffer = Buffer.from(imageBlock.source.data, 'base64');
    const hash = crypto.createHash('md5').update(buffer).digest('hex');
    const ext = (imageBlock.source.media_type || 'image/png').split('/')[1] || 'png';
    const storedFilename = `${hash}.${ext}`;
    const filePath = path.join(dbModule.DEFAULT_IMAGES_DIR, storedFilename);

    // Dedup: only write if file doesn't exist
    if (!fs.existsSync(filePath)) {
      fs.writeFileSync(filePath, buffer);
    }

    // Record in images table (with NULL prompt_id for harvested images)
    const d = dbModule.getDb();
    const existing = d.prepare('SELECT id FROM images WHERE file_path = ?').get(filePath);
    if (existing) return { id: existing.id, filename: storedFilename, path: filePath };

    const result = d.prepare(
      'INSERT INTO images (prompt_id, filename, mime_type, file_path) VALUES (NULL, ?, ?, ?)'
    ).run(storedFilename, imageBlock.source.media_type || 'image/png', filePath);
    return { id: result.lastInsertRowid, filename: storedFilename, path: filePath };
  } catch (e) {
    if (/database is locked/i.test(e.message || '')) return null;
    console.error('[harvest] Failed to save image:', e.message);
    return null;
  }
}

// Harvest a single session into prompt_executions
function harvestSession(filePath, projectPath) {
  const parsed = parseFullSession(filePath);
  if (!parsed || parsed.messages.length === 0) return 0;

  const d = dbModule.getDb();
  const insertExec = d.prepare(`
    INSERT OR IGNORE INTO prompt_executions
    (session_id, message_text, message_index, role, tool_uses, image_refs, project_path, cwd, source, executed_at)
    VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'harvested', ?)
  `);

  let count = 0;
  // Insert in BOUNDED BATCHES, each its OWN transaction. A long active session can carry
  // thousands of messages; wrapping them all in a single transaction held the db-owner
  // write lock for tens of seconds (observed 34–59s), which busy-waits CTM's main-loop
  // synchronous reads and freezes the loop — long enough that the Wall-E MCP readiness
  // probe times out 3× and CTM raises a false "MCP never became ready". Committing in
  // small batches releases the lock between batches so the main loop + other writers can
  // interleave. INSERT OR IGNORE keeps it idempotent across batches/runs.
  const BATCH = Math.max(1, Number(process.env.CTM_HARVEST_MSG_BATCH || 250));
  const insertBatch = d.transaction((slice) => {
    for (const msg of slice) {
      // Save images and collect refs
      const imageRefs = [];
      if (msg.images) {
        for (const imgBlock of msg.images) {
          const saved = saveHarvestedImage(imgBlock, parsed.sessionId);
          if (saved) imageRefs.push(saved);
        }
      }

      const toolUsesJson = msg.toolUses ? JSON.stringify(msg.toolUses) : '[]';

      try {
        insertExec.run(
          parsed.sessionId,
          msg.text.slice(0, 10000), // Limit text size
          msg.index,
          msg.role,
          toolUsesJson,
          JSON.stringify(imageRefs),
          projectPath,
          parsed.cwd || projectPath,
          msg.timestamp || null
        );
        count++;
      } catch { /* ignore dups */ }

      // Detect /skill-name patterns in user messages
      if (msg.role === 'user') {
        const skillMatches = msg.text.match(/(?:^|\s)\/([\w-]+)/g);
        if (skillMatches) {
          for (const match of skillMatches) {
            const skillName = match.trim().slice(1); // remove leading /
            if (skillName.length >= 2 && skillName.length <= 50) {
              try { dbModule.upsertSkillUsage(skillName); } catch { /* ignore */ }
            }
          }
        }
      }
    }
  });

  const msgs = parsed.messages;
  for (let i = 0; i < msgs.length; i += BATCH) {
    insertBatch(msgs.slice(i, i + BATCH));
  }
  return count;
}

const HARVEST_CURSOR_SETTING = 'prompt_harvest_incremental_cursor';
const LIFECYCLE_REFRESH_DEFAULT_INTERVAL_MS = 6 * 60 * 60 * 1000;

function getHarvestCursor(total) {
  let cursor = 0;
  try { cursor = Number(dbModule.getSetting(HARVEST_CURSOR_SETTING, 0)); } catch {}
  if (!Number.isFinite(cursor) || cursor < 0) return 0;
  if (total <= 0) return 0;
  return Math.min(Math.floor(cursor), total - 1);
}

function setHarvestCursor(cursor) {
  try { dbModule.setSetting(HARVEST_CURSOR_SETTING, Math.max(0, Math.floor(cursor || 0))); } catch {}
}

function parseTimestampMs(value) {
  if (!value) return 0;
  const text = String(value).trim();
  if (!text) return 0;
  const normalized = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/.test(text)
    ? text.replace(' ', 'T') + (text.endsWith('Z') ? '' : 'Z')
    : text;
  const ms = Date.parse(normalized);
  return Number.isFinite(ms) ? ms : 0;
}

function getLifecycleRefreshIntervalMs() {
  const raw = Number(process.env.CTM_LIFECYCLE_REFRESH_INTERVAL_MS || 0);
  return Number.isFinite(raw) && raw > 0 ? raw : LIFECYCLE_REFRESH_DEFAULT_INTERVAL_MS;
}

function shouldRefreshLifecycleAfterHarvest(state, harvestResult) {
  if (harvestResult?.partial) return false;
  if ((harvestResult?.messagesHarvested || 0) > 0) return true;
  const lastRefreshMs = parseTimestampMs(state?.last_lifecycle_refresh_at);
  return !lastRefreshMs || (Date.now() - lastRefreshMs) >= getLifecycleRefreshIntervalMs();
}

// Incremental harvest — only scan modified sessions
function runHarvest(options = {}) {
  const { scope = 'all', maxDays, projectFilter, onProgress } = options;
  const maxFiles = Math.max(0, Number(options.maxFiles || 0));
  const state = getHarvestState();
  const lastScan = state?.last_scan_at ? new Date(state.last_scan_at + 'Z') : null;
  const allFiles = sessionUtils.getAllSessionFiles()
    .sort((a, b) => String(a.filePath || '').localeCompare(String(b.filePath || '')));

  // Filter by scope
  let filesToScan = allFiles;
  if (scope === 'incremental' && lastScan) {
    filesToScan = allFiles.filter(f => {
      try {
        const mtime = fs.statSync(f.filePath).mtime;
        return mtime > lastScan;
      } catch { return false; }
    });
  }
  if (maxDays) {
    const cutoff = new Date(Date.now() - maxDays * 86400000);
    filesToScan = filesToScan.filter(f => {
      try { return fs.statSync(f.filePath).mtime > cutoff; } catch { return false; }
    });
  }
  if (projectFilter) {
    filesToScan = filesToScan.filter(f => f.projectPath.includes(projectFilter));
  }

  const eligibleFiles = filesToScan.length;
  let partial = false;
  let cursor = 0;
  let nextCursor = 0;
  if (scope === 'incremental' && maxFiles > 0 && eligibleFiles > maxFiles) {
    cursor = getHarvestCursor(eligibleFiles);
    nextCursor = Math.min(cursor + maxFiles, eligibleFiles);
    filesToScan = filesToScan.slice(cursor, nextCursor);
    partial = nextCursor < eligibleFiles;
  }

  let totalHarvested = 0;
  for (let i = 0; i < filesToScan.length; i++) {
    const f = filesToScan[i];
    try {
      const count = harvestSession(f.filePath, f.projectPath);
      totalHarvested += count;
      if (onProgress && i % 10 === 0) {
        onProgress({ current: i + 1, total: filesToScan.length, harvested: totalHarvested });
      }
    } catch (e) {
      console.error(`[harvest] Error processing ${f.filePath}:`, e.message);
    }
  }

  if (scope === 'incremental' && maxFiles > 0 && eligibleFiles > maxFiles) {
    setHarvestCursor(partial ? nextCursor : 0);
  }

  const stateUpdates = {
    files_scanned: (state?.files_scanned || 0) + filesToScan.length,
    prompts_harvested: (state?.prompts_harvested || 0) + totalHarvested,
  };
  if (!partial) {
    stateUpdates.last_scan_at = new Date().toISOString().replace('T', ' ').replace('Z', '');
  }
  updateHarvestState(stateUpdates);

  const harvestResult = {
    filesScanned: filesToScan.length,
    totalFiles: allFiles.length,
    eligibleFiles,
    messagesHarvested: totalHarvested,
    partial,
    nextCursor: partial ? nextCursor : 0,
  };

  // Auto-refresh lifecycle statuses after a completed harvest batch. This
  // matching pass compares all prompt templates against all harvested user
  // messages, so running it on every idle timer cycle can burn a CTM worker
  // thread for tens of seconds even when no session content changed.
  const lifecycle = shouldRefreshLifecycleAfterHarvest(state, harvestResult)
    ? refreshLifecycleStatuses()
    : {
        skipped: true,
        reason: partial ? 'partial_harvest_batch' : 'not_due',
        next_due_ms: getLifecycleRefreshIntervalMs(),
      };

  return {
    ...harvestResult,
    lifecycle,
  };
}

// Get harvest preview (stats without scanning)
function getHarvestPreview() {
  const allFiles = sessionUtils.getAllSessionFiles();
  const state = getHarvestState();
  const lastScan = state?.last_scan_at ? new Date(state.last_scan_at + 'Z') : null;

  let totalSize = 0;
  let modifiedSince = 0;
  let oldest = null;
  let newest = null;
  const byProject = {};

  for (const f of allFiles) {
    try {
      const stat = fs.statSync(f.filePath);
      totalSize += stat.size;
      if (lastScan && stat.mtime > lastScan) modifiedSince++;
      if (!oldest || stat.mtime < oldest) oldest = stat.mtime;
      if (!newest || stat.mtime > newest) newest = stat.mtime;
      const proj = f.projectPath.replace(/^\/Users\/[^/]+\//, '~/');
      byProject[proj] = (byProject[proj] || 0) + 1;
    } catch {}
  }

  return {
    totalSessions: allFiles.length,
    totalSizeMB: (totalSize / 1024 / 1024).toFixed(1),
    modifiedSinceLastScan: modifiedSince,
    dateRange: { oldest: oldest?.toISOString(), newest: newest?.toISOString() },
    projects: Object.entries(byProject).sort((a, b) => b[1] - a[1]).map(([name, count]) => ({ name, count })),
    lastScan: state?.last_scan_at || null,
    totalHarvested: state?.prompts_harvested || 0,
  };
}

// ============================================================
// Autocomplete Engine — local fuzzy search on harvested data
// ============================================================

let _autocompleteCache = null;
let _autocompleteCacheTime = 0;
const CACHE_TTL = 60000; // Refresh cache every 60s

function buildAutocompleteIndex() {
  let rows;
  try {
    const d = dbModule.getDb();
    rows = d.prepare(`
      SELECT message_text, tool_uses, session_id, project_path, role, executed_at
      FROM prompt_executions
      ORDER BY executed_at DESC
      LIMIT 5000
    `).all();
  } catch (err) {
    console.error('[autocomplete] Failed to query prompt_executions:', err.code || err.message);
    return [];
  }

  const entries = [];
  for (const row of rows) {
    // User messages → autocomplete suggestions
    if (row.role === 'user') {
      // Strip XML/HTML tags and get first meaningful line
      const cleanText = row.message_text.replace(/<[^>]+>/g, '').trim();
      const firstLine = cleanText.split('\n').find(l => l.trim().length > 5)?.trim() || '';
      if (firstLine.length > 10) {
        entries.push({
          text: firstLine.slice(0, 200),
          fullText: cleanText.slice(0, 1000),
          type: 'history',
          sessionId: row.session_id,
          project: row.project_path,
          at: row.executed_at,
        });
      }
    }

    // Assistant tool uses → context for autocomplete
    if (row.role === 'assistant' && row.tool_uses && row.tool_uses !== '[]') {
      try {
        const tools = JSON.parse(row.tool_uses);
        for (const t of tools) {
          if (t.input_summary) {
            entries.push({
              text: `[${t.name}] ${t.input_summary}`,
              fullText: `Claude used ${t.name}: ${t.input_summary}`,
              type: 'tool_context',
              sessionId: row.session_id,
              project: row.project_path,
              at: row.executed_at,
            });
          }
        }
      } catch {}
    }
  }

  // Also include prompts from the library
  try {
    const d = dbModule.getDb();
    const prompts = d.prepare('SELECT id, title, content FROM prompts ORDER BY updated_at DESC LIMIT 200').all();
    for (const p of prompts) {
      entries.push({
        text: p.title,
        fullText: (p.content || '').slice(0, 1000),
        type: 'library',
        promptId: p.id,
      });
    }
  } catch {}

  return entries;
}

function getAutocompleteIndex() {
  const now = Date.now();
  if (_autocompleteCache && now - _autocompleteCacheTime < CACHE_TTL) {
    return _autocompleteCache;
  }
  _autocompleteCache = buildAutocompleteIndex();
  _autocompleteCacheTime = now;
  return _autocompleteCache;
}

function invalidateAutocompleteCache() {
  _autocompleteCache = null;
}

// Fuzzy search autocomplete entries
function searchAutocomplete(query, limit = 8) {
  if (!query || query.length < 2) return [];
  const index = getAutocompleteIndex();
  const queryLower = query.toLowerCase().trim();
  const queryWords = queryLower.split(/\s+/).filter(w => w.length >= 2);
  if (queryWords.length === 0) return [];

  const scored = [];
  const seen = new Set();
  const nonPrefix = []; // collect non-prefix for BM25

  for (const entry of index) {
    const textLower = entry.text.toLowerCase();
    const dedup = textLower.slice(0, 150);
    if (seen.has(dedup)) continue;
    seen.add(dedup);

    // Prefix match (strongest signal — keep as fast path)
    if (textLower.startsWith(queryLower)) {
      scored.push({ ...entry, score: 100 + (entry.type === 'library' ? 2 : 0) });
      continue;
    }
    nonPrefix.push(entry);
  }

  // BM25 rank the non-prefix matches
  if (_bm25 && nonPrefix.length > 0) {
    const docs = nonPrefix.map(e => ({ ...e, text: (e.fullText || e.text) }));
    const ranked = _bm25.bm25Rerank(docs, query);
    for (const r of ranked) {
      if (r._bm25 > 0) {
        scored.push({ ...r, score: Math.min(r._bm25 * 10, 90) + (r.type === 'library' ? 2 : 0) });
      }
    }
  } else {
    // Fallback: original word-match scoring
    for (const entry of nonPrefix) {
      const textLower = entry.text.toLowerCase();
      const fullLower = (entry.fullText || '').toLowerCase();
      let allMatch = true;
      let wordScore = 0;
      for (const w of queryWords) {
        if (textLower.includes(w)) wordScore += 10;
        else if (fullLower.includes(w)) wordScore += 5;
        else { allMatch = false; break; }
      }
      if (allMatch && wordScore > 0) {
        if (entry.type === 'library') wordScore += 2;
        scored.push({ ...entry, score: wordScore });
      }
    }
  }

  scored.sort((a, b) => b.score - a.score || (b.at || '').localeCompare(a.at || ''));
  return scored.slice(0, limit).map(({ score, _bm25, ...entry }) => entry);
}

function _augmentWithEffectiveness(results) {
  try { ensureEffectivenessTable(); } catch {} // init once, not per-row
  return results.map(r => {
    try {
      const text = r.fullText || r.text || '';
      if (text.length < 20) return r;
      const eff = _getEffectivenessQuick(text);
      if (eff && eff.sample_count >= 1) {
        r.avgOutcome = eff.avg_outcome;
        r.sampleCount = eff.sample_count;
      }
    } catch {}
    return r;
  });
}

// Find similar past prompts (more thorough than autocomplete)
function findSimilarPrompts(text, limit = 5, excludePromptId = null) {
  if (!text || text.length < 20) return [];
  const index = getAutocompleteIndex();
  const queryLower = text.toLowerCase().trim();

  // Filter candidates
  const candidates = [];
  const seen = new Set();
  for (const entry of index) {
    if (entry.type === 'tool_context') continue;
    if (excludePromptId && entry.type === 'library' && entry.promptId === excludePromptId) continue;
    const fullLower = (entry.fullText || entry.text).toLowerCase();
    if (fullLower.trim() === queryLower || queryLower.startsWith(fullLower.trim())) continue;
    const key = fullLower.slice(0, 100);
    if (seen.has(key)) continue;
    seen.add(key);
    candidates.push(entry);
  }

  if (candidates.length === 0) return [];

  // BM25 ranking
  if (_bm25) {
    const docs = candidates.map(e => ({ ...e, text: (e.fullText || e.text) }));
    const ranked = _bm25.bm25Rerank(docs, text);
    const results = ranked.filter(r => r._bm25 > 0).slice(0, limit).map(({ _bm25, ...entry }) => entry);
    return _augmentWithEffectiveness(results);
  }

  // Fallback: original word-overlap scoring
  const queryWords = text.toLowerCase().split(/\s+/).filter(w => w.length > 3);
  if (queryWords.length === 0) return [];
  const scored = [];
  for (const entry of candidates) {
    const fullLower = (entry.fullText || entry.text).toLowerCase();
    let matchCount = 0;
    for (const w of queryWords) {
      if (fullLower.includes(w)) matchCount++;
    }
    const matchRatio = matchCount / queryWords.length;
    if (matchRatio >= 0.3) {
      scored.push({ ...entry, matchRatio, matchCount });
    }
  }
  scored.sort((a, b) => b.matchRatio - a.matchRatio || b.matchCount - a.matchCount);
  return _augmentWithEffectiveness(scored.slice(0, limit));
}

// ============================================================
// Pattern Detection — identify reusable prompt patterns
// ============================================================

function detectPatterns() {
  const d = dbModule.getDb();
  const userMsgs = d.prepare(`
    SELECT message_text, project_path, session_id, executed_at
    FROM prompt_executions
    WHERE role = 'user' AND length(message_text) > 20
    ORDER BY executed_at DESC
    LIMIT 2000
  `).all();

  // Normalize and group
  const normalized = new Map(); // normalized_text -> [{original, session, project, at}]

  for (const msg of userMsgs) {
    const norm = normalizePrompt(msg.message_text);
    if (norm.length < 15) continue;

    if (!normalized.has(norm)) normalized.set(norm, []);
    normalized.get(norm).push({
      original: msg.message_text.slice(0, 500),
      session: msg.session_id,
      project: msg.project_path,
      at: msg.executed_at,
    });
  }

  // Find patterns with 2+ occurrences
  const patterns = [];
  for (const [norm, instances] of normalized) {
    if (instances.length >= 2) {
      patterns.push({
        normalizedText: norm.slice(0, 200),
        frequency: instances.length,
        example: instances[0].original,
        sessions: [...new Set(instances.map(i => i.session))].slice(0, 5),
        projects: [...new Set(instances.map(i => i.project))],
        lastUsed: instances[0].at,
      });
    }
  }

  patterns.sort((a, b) => b.frequency - a.frequency);

  // Save to DB
  const upsert = d.prepare(`
    INSERT INTO prompt_patterns (normalized_text, frequency, example_text, sessions, projects, last_used_at)
    VALUES (?, ?, ?, ?, ?, ?)
    ON CONFLICT(normalized_text) DO UPDATE SET
      frequency = excluded.frequency,
      example_text = excluded.example_text,
      sessions = excluded.sessions,
      projects = excluded.projects,
      last_used_at = excluded.last_used_at,
      updated_at = datetime('now')
  `);

  const txn = d.transaction(() => {
    for (const p of patterns.slice(0, 100)) {
      upsert.run(
        p.normalizedText,
        p.frequency,
        p.example,
        JSON.stringify(p.sessions),
        JSON.stringify(p.projects),
        p.lastUsed
      );
    }
  });
  txn();

  return patterns.slice(0, 50);
}

// Normalize a prompt for pattern matching
function normalizePrompt(text) {
  return text
    .toLowerCase()
    .replace(/['"]/g, '')
    .replace(/\b(the|a|an|is|are|was|were|be|been|being|have|has|had|do|does|did|will|would|could|should|can|may|might|shall|this|that|these|those|it|its|my|your|our|their)\b/g, ' ')
    .replace(/[^a-z0-9\s]/g, ' ')
    .replace(/\s+/g, ' ')
    .trim();
}

// ============================================================
// Prompt Copilot — AI-powered suggestions
// ============================================================

async function getCopilotSuggestion(currentText, context = {}) {
  const baseUrl = process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com';
  const apiKey = process.env.ANTHROPIC_API_KEY || '';
  // Build context from similar prompts and recent session activity
  // Exclude the current prompt being edited so it doesn't suggest itself
  const excludeId = context.promptId ? Number(context.promptId) : null;
  const similar = findSimilarPrompts(currentText, 3, excludeId);

  // Local fallback when no API key — suggest based on similar prompts
  if (!apiKey) {
    if (similar.length === 0) return null;
    const best = similar[0];
    const suggestion = similar.length === 1
      ? `Similar prompt found: "${best.text.slice(0, 120)}"`
      : `${similar.length} similar prompts found. Most relevant: "${best.text.slice(0, 120)}"`;
    return { suggestion, improved: best.fullText || best.text, quality: 'unknown' };
  }
  const similarContext = similar.length > 0
    ? '\n\nSimilar prompts from history:\n' + similar.map(s =>
      `- "${s.text}" (${s.type === 'library' ? 'from library' : 'used ' + (s.at || 'before')})`
    ).join('\n')
    : '';

  const prompt = `You are a Prompt Copilot — helping a developer write better prompts for Claude Code.

The user is currently writing this prompt:
"${currentText.slice(0, 1000)}"
${similarContext}
${context.projectPath ? `\nProject: ${context.projectPath}` : ''}

Give a brief suggestion (2-3 sentences max) to improve this prompt. Focus on:
- Clarity: Is the intent clear?
- Specificity: Are file paths, function names, or scope mentioned?
- Efficiency: Could this be more concise without losing meaning?

If the prompt is already good, say so briefly. If you have a concrete improvement, show the improved version.

Respond in JSON: {"suggestion": "...", "improved": "..." or null, "quality": "good"|"needs_work"|"vague"}`;

  // Parse custom headers
  let customHeaders = {};
  try {
    const headerStr = process.env.ANTHROPIC_CUSTOM_HEADERS || '';
    if (headerStr) {
      for (const pair of headerStr.split(',')) {
        const [k, ...v] = pair.split(':');
        if (k && v.length) customHeaders[k.trim()] = v.join(':').trim();
      }
    }
  } catch {}

  try {
    const res = await fetch(`${baseUrl}/messages`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'x-api-key': apiKey,
        'anthropic-version': '2023-06-01',
        ...customHeaders,
      },
      body: JSON.stringify({
        model: 'claude-haiku-4-5-20251001',
        max_tokens: 300,
        messages: [{ role: 'user', content: prompt }],
      }),
    });

    if (!res.ok) return null;
    const data = await res.json();
    const text = data.content?.[0]?.text || '';
    const match = text.match(/\{[\s\S]*\}/);
    if (!match) return { suggestion: text.slice(0, 300), improved: null, quality: 'unknown' };
    return JSON.parse(match[0]);
  } catch (e) {
    console.error('[copilot] Error:', e.message);
    return null;
  }
}

// Local copilot fallback — keyword search over prompt history (no API key needed)
function copilotChatLocal(userMessage, recentPrompts, patterns) {
  const msg = userMessage.toLowerCase();

  // Pattern queries
  if (msg.includes('pattern') || msg.includes('common') || msg.includes('frequent') || msg.includes('most')) {
    if (patterns.length === 0) return { reply: 'No patterns detected yet. Run "Harvest" first to scan your sessions, then "Re-detect" in the Patterns view.', error: false };
    const lines = patterns.slice(0, 8).map((p, i) =>
      `${i + 1}. "${(p.example_text || '').slice(0, 100)}" — used ${p.frequency}x`
    );
    return { reply: `Your top prompt patterns:\n\n${lines.join('\n')}\n\nTotal: ${patterns.length} patterns detected.`, error: false };
  }

  // Search queries
  if (msg.includes('search') || msg.includes('find') || msg.includes('about') || msg.includes('last') || msg.includes('when') || msg.includes('did i')) {
    const keywords = msg.replace(/[^a-z0-9 ]/g, '').split(/\s+/).filter(w => w.length > 3 && !['search','find','about','what','when','last','that','this','have','with','from','prompt'].includes(w));
    if (keywords.length === 0) return { reply: 'Try asking about a specific topic, e.g. "find prompts about authentication" or "when did I ask about refactoring?"', error: false };
    const matches = recentPrompts.filter(p => {
      const t = p.message_text.toLowerCase();
      return keywords.some(k => t.includes(k));
    }).slice(0, 5);
    if (matches.length === 0) return { reply: `No recent prompts found matching: ${keywords.join(', ')}. Try different keywords or run Harvest to scan more sessions.`, error: false };
    const lines = matches.map(p =>
      `- [${p.executed_at?.slice(0, 10) || '?'}] "${p.message_text.slice(0, 120)}..." (${p.project_path || 'unknown project'})`
    );
    return { reply: `Found ${matches.length} matching prompts:\n\n${lines.join('\n')}`, error: false };
  }

  // Stats
  if (msg.includes('stat') || msg.includes('how many') || msg.includes('count') || msg.includes('total')) {
    const d = dbModule.getDb();
    const total = d.prepare('SELECT COUNT(*) as c FROM prompt_executions').get()?.c || 0;
    const userCount = d.prepare("SELECT COUNT(*) as c FROM prompt_executions WHERE role = 'user'").get()?.c || 0;
    const projects = d.prepare('SELECT COUNT(DISTINCT project_path) as c FROM prompt_executions').get()?.c || 0;
    return { reply: `Prompt stats:\n- Total harvested messages: ${total}\n- User prompts: ${userCount}\n- Projects: ${projects}\n- Patterns detected: ${patterns.length}`, error: false };
  }

  // Quality/improvement queries
  if (msg.includes('improve') || msg.includes('better') || msg.includes('quality') || msg.includes('how can i')) {
    const currentText = recentPrompts[0]?.message_text || '';
    if (currentText.length > 20) {
      const quality = assessPromptQuality(currentText);
      const eff = getEffectivenessForPrompt(currentText);
      let reply = `Quality assessment: ${quality.quality}\n${quality.hint}\n`;
      if (eff) {
        reply += `\nHistorical effectiveness: ${eff.avg_outcome > 0 ? '+' : ''}${eff.avg_outcome.toFixed(2)} across ${eff.sample_count} uses.`;
        if (eff.best_variant && eff.best_variant !== currentText) {
          reply += `\n\nA more effective variant:\n"${eff.best_variant.slice(0, 200)}"`;
        }
      }
      return { reply, error: false };
    }
    return { reply: 'Open a prompt first, then ask "How can I improve this prompt?"', error: false };
  }

  // Effectiveness/outcome queries
  if (msg.includes('effective') || msg.includes('success') || msg.includes('outcome') || (msg.includes('work') && msg.includes('well'))) {
    try {
      const d = dbModule.getDb();
      ensureEffectivenessTable();
      const top = d.prepare('SELECT normalized_text, avg_outcome, sample_count FROM prompt_effectiveness WHERE sample_count >= 3 ORDER BY avg_outcome DESC LIMIT 5').all();
      const bottom = d.prepare('SELECT normalized_text, avg_outcome, sample_count FROM prompt_effectiveness WHERE sample_count >= 3 ORDER BY avg_outcome ASC LIMIT 5').all();
      let reply = 'Prompt effectiveness analysis:\n\n';
      if (top.length > 0) {
        reply += 'Most effective prompts:\n' + top.map((r, i) => `${i + 1}. "${r.normalized_text.slice(0, 80)}" (${r.avg_outcome > 0 ? '+' : ''}${r.avg_outcome.toFixed(2)}, ${r.sample_count} uses)`).join('\n');
      }
      if (bottom.length > 0) {
        reply += '\n\nLeast effective:\n' + bottom.map((r, i) => `${i + 1}. "${r.normalized_text.slice(0, 80)}" (${r.avg_outcome.toFixed(2)}, ${r.sample_count} uses)`).join('\n');
      }
      return { reply: reply || 'No effectiveness data yet. Run "Harvest" and then update effectiveness.', error: false };
    } catch { return { reply: 'Effectiveness data not available. Run harvest first.', error: false }; }
  }

  // Workflow/trajectory queries
  if (msg.includes('workflow') || msg.includes('after') || msg.includes('follow') || msg.includes('sequence') || msg.includes('next')) {
    const keywords = msg.replace(/[^a-z0-9 ]/g, '').split(/\s+/).filter(w => w.length > 3 && !['workflow','after','follow','sequence','next','what','usually','typically','then'].includes(w));
    if (keywords.length > 0) {
      const matches = recentPrompts.filter(p => {
        const t = p.message_text.toLowerCase();
        return keywords.some(k => t.includes(k));
      }).slice(0, 3);
      if (matches.length > 0) {
        const d = dbModule.getDb();
        let reply = 'Workflow analysis:\n';
        for (const m of matches) {
          const nextMsgs = d.prepare("SELECT message_text FROM prompt_executions WHERE session_id = ? AND message_index > ? AND role = 'user' ORDER BY message_index ASC LIMIT 3").all(m.session_id, m.message_index || 0);
          if (nextMsgs.length > 0) {
            reply += `\nAfter "${m.message_text.slice(0, 60)}...":\n`;
            reply += nextMsgs.map((n, i) => `  ${i + 1}. "${n.message_text.slice(0, 80)}"`).join('\n') + '\n';
          }
        }
        return { reply, error: false };
      }
    }
    return { reply: 'Try "What\'s my workflow for [topic]?" with a specific topic keyword.', error: false };
  }

  // Help / default
  return { reply: `I can help with your prompt history (no API key needed for local features):\n\n- "What are my most common patterns?" — show detected patterns\n- "Find prompts about [topic]" — search your history\n- "How many prompts?" — show stats\n\nFor AI-powered suggestions, set ANTHROPIC_API_KEY in the server environment.`, error: false };
}

// Copilot chat — answer questions about prompt history
async function copilotChat(userMessage, conversationHistory = []) {
  const baseUrl = process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com';
  const apiKey = process.env.ANTHROPIC_API_KEY || '';
  // Build context from prompt history
  const d = dbModule.getDb();
  const recentPrompts = d.prepare(`
    SELECT message_text, project_path, role, tool_uses, executed_at, session_id, message_index
    FROM prompt_executions
    WHERE role = 'user'
    ORDER BY executed_at DESC
    LIMIT 20
  `).all();

  const patterns = d.prepare('SELECT * FROM prompt_patterns ORDER BY frequency DESC LIMIT 10').all();

  // Local-only fallback when no API key
  if (!apiKey) {
    return copilotChatLocal(userMessage, recentPrompts, patterns);
  }

  const historyContext = recentPrompts.map(p =>
    `[${p.executed_at}] ${p.message_text.slice(0, 150)}`
  ).join('\n');

  const patternContext = patterns.map(p =>
    `"${p.example_text?.slice(0, 100)}" (used ${p.frequency}x)`
  ).join('\n');

  // Parse custom headers
  let customHeaders = {};
  try {
    const headerStr = process.env.ANTHROPIC_CUSTOM_HEADERS || '';
    if (headerStr) {
      for (const pair of headerStr.split(',')) {
        const [k, ...v] = pair.split(':');
        if (k && v.length) customHeaders[k.trim()] = v.join(':').trim();
      }
    }
  } catch {}

  const systemPrompt = `You are Prompt Copilot — a helpful assistant that knows the user's prompting history with Claude Code.

Recent prompts (newest first):
${historyContext || '(no history yet)'}

Frequently used patterns:
${patternContext || '(no patterns detected yet)'}

Help the user with:
- Finding past prompts ("what did I ask about auth last week?")
- Improving prompts ("how can I make this prompt clearer?")
- Suggesting approaches ("what's the best way to ask Claude to refactor X?")
- Pattern insights ("what kinds of prompts do I use most?")

Be concise and practical. Reference specific past prompts when relevant.`;

  const messages = [
    ...conversationHistory.slice(-10),
    { role: 'user', content: userMessage },
  ];

  try {
    const res = await fetch(`${baseUrl}/messages`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'x-api-key': apiKey,
        'anthropic-version': '2023-06-01',
        ...customHeaders,
      },
      body: JSON.stringify({
        model: 'claude-haiku-4-5-20251001',
        max_tokens: 500,
        system: systemPrompt,
        messages,
      }),
    });

    if (!res.ok) {
      const errText = await res.text();
      return { reply: `API error: ${res.status}`, error: true };
    }
    const data = await res.json();
    return { reply: data.content?.[0]?.text || 'No response', error: false };
  } catch (e) {
    return { reply: `Error: ${e.message}`, error: true };
  }
}

// ============================================================
// Automated Lifecycle Tracking
// ============================================================

// Match library prompts against harvested session text to auto-promote lifecycle status.
// Rules:
//   draft → used: prompt text found in any session (even copy-pasted in terminal)
//   used → frequent: found in 3+ distinct sessions
//   (stale): not used in 30+ days → flagged for archiving
// Never auto-demotes: template and archived are manual statuses.
function refreshLifecycleStatuses() {
  const d = dbModule.getDb();
  const prompts = d.prepare(`SELECT id, title, content, lifecycle_status FROM prompts`).all();
  if (prompts.length === 0) return { updated: [], stale: [], refreshedAt: new Date().toISOString() };

  // Gather all user messages from prompt_executions for matching
  const allExecs = d.prepare(`
    SELECT DISTINCT session_id, message_text, executed_at
    FROM prompt_executions
    WHERE role = 'user' AND length(message_text) > 20
  `).all();

  // Also gather explicit sends from prompt_usage
  const allUsage = d.prepare(`SELECT prompt_id, session_id, used_at FROM prompt_usage`).all();
  const usageByPrompt = new Map();
  for (const u of allUsage) {
    if (!usageByPrompt.has(u.prompt_id)) usageByPrompt.set(u.prompt_id, new Set());
    usageByPrompt.get(u.prompt_id).add(u.session_id);
  }

  const updated = [];
  const stale = [];
  const now = Date.now();
  const STALE_DAYS = 30;

  // Phase 1 — MATCH (read + CPU only, NO write lock). This nested loop is
  // O(prompts × execs × words) substring matching over arrays already loaded
  // above; it touches no DB and must not hold the SQLite write lock. It used to
  // run inside d.transaction(), which pinned the write lock for 80-90s on the
  // db-owner worker and fail-fasted every main-thread write — most visibly the
  // hotkey screenshot INSERT ("took screenshot, nothing happens"). Computing the
  // status changes here and applying them in a separate short transaction below
  // drops the lock hold from ~90s to milliseconds.
  for (const prompt of prompts) {
    const currentStatus = prompt.lifecycle_status || 'draft';
    // Skip manually managed statuses
    if (currentStatus === 'template' || currentStatus === 'archived') continue;

    const content = (prompt.content || '').trim();
    if (content.length < 15) continue;

    // Find sessions where this prompt's text appears
    const matchedSessions = new Set();
    let latestMatch = null;

    // 1. Check explicit sends via prompt_usage
    const explicitSessions = usageByPrompt.get(prompt.id);
    if (explicitSessions) {
      for (const sid of explicitSessions) matchedSessions.add(sid);
    }

    // 2. Text-match against harvested session messages
    const contentWords = content.toLowerCase().split(/\s+/).filter(w => w.length > 4);
    if (contentWords.length >= 3) {
      const contentNorm = normalizePrompt(content);
      const minMatchCount = Math.max(3, Math.ceil(contentWords.length * 0.5));

      for (const exec of allExecs) {
        if (matchedSessions.has(exec.session_id) && latestMatch) continue;

        const execLower = exec.message_text.toLowerCase();

        let matchCount = 0;
        for (const w of contentWords) {
          if (execLower.includes(w)) matchCount++;
        }

        const matched = matchCount >= minMatchCount;
        let substringMatched = false;
        if (!matched) {
          const execNorm = normalizePrompt(exec.message_text);
          substringMatched = execNorm.length >= 30 && (execNorm.includes(contentNorm.slice(0, 80)) || contentNorm.includes(execNorm.slice(0, 80)));
        }

        if (matched || substringMatched) {
          matchedSessions.add(exec.session_id);
          if (!latestMatch || (exec.executed_at && exec.executed_at > latestMatch)) {
            latestMatch = exec.executed_at;
          }
        }
      }
    }

    const sessionCount = matchedSessions.size;
    let newStatus = currentStatus;

    if (sessionCount >= 3 && (currentStatus === 'draft' || currentStatus === 'used')) {
      newStatus = 'frequent';
    } else if (sessionCount >= 1 && currentStatus === 'draft') {
      newStatus = 'used';
    }

    if (newStatus !== currentStatus) {
      updated.push({ id: prompt.id, title: prompt.title, from: currentStatus, to: newStatus, sessions: sessionCount });
    }

    // Check staleness (for used/frequent prompts only)
    if ((newStatus === 'used' || newStatus === 'frequent') && latestMatch) {
      const lastUsedDate = new Date(latestMatch);
      const daysSince = (now - lastUsedDate.getTime()) / 86400000;
      if (daysSince > STALE_DAYS) {
        stale.push({ id: prompt.id, title: prompt.title, status: newStatus, daysSinceUsed: Math.round(daysSince), lastUsed: latestMatch });
      }
    }
  }

  // Phase 2 — APPLY (write lock held only for the handful of status UPDATEs).
  if (updated.length > 0) {
    const updateStmt = d.prepare(`UPDATE prompts SET lifecycle_status = ?, updated_at = datetime('now') WHERE id = ?`);
    const applyTxn = d.transaction((rows) => {
      for (const u of rows) updateStmt.run(u.to, u.id);
    });
    applyTxn(updated);
  }

  // Update last_lifecycle_refresh_at watermark
  const refreshedAt = new Date().toISOString().replace('T', ' ').replace('Z', '');
  updateHarvestState({ last_lifecycle_refresh_at: refreshedAt });

  if (updated.length > 0) {
    console.log(`[lifecycle] Auto-updated ${updated.length} prompt(s):`, updated.map(u => `${u.title}: ${u.from}→${u.to}`).join(', '));
  }
  if (stale.length > 0) {
    console.log(`[lifecycle] ${stale.length} stale prompt(s) (unused ${STALE_DAYS}+ days)`);
  }

  return { updated, stale, refreshedAt };
}

// Run incremental harvest + lifecycle refresh (used by timer and startup)
function runIncrementalLifecycleRefresh() {
  try {
    const maxFiles = Math.max(1, Number(process.env.CTM_HARVEST_MAX_FILES_PER_RUN || 2));
    const harvestResult = runHarvest({ scope: 'incremental', maxFiles });
    invalidateAutocompleteCache();
    const suffix = harvestResult.partial
      ? ` (partial, next ${harvestResult.nextCursor}/${harvestResult.eligibleFiles})`
      : '';
    console.log(`[lifecycle-timer] Harvested ${harvestResult.messagesHarvested} messages from ${harvestResult.filesScanned} files${suffix}`);
    return harvestResult.lifecycle;
  } catch (e) {
    console.error('[lifecycle-timer] Error:', e.message);
    return null;
  }
}

// ============================================================
// Prompt-Session Mapping — bidirectional links
// ============================================================

function getSessionsForPrompt(promptId, limit = 20) {
  const d = dbModule.getDb();

  // 1. Explicit sends via prompt_usage
  const usage = d.prepare(`
    SELECT session_id, used_at as at, 'sent' as source
    FROM prompt_usage
    WHERE prompt_id = ?
    ORDER BY used_at DESC
  `).all(promptId);

  const seen = new Set(usage.map(u => u.session_id));
  const results = [...usage];

  // 2. Text-match against harvested prompt_executions (same logic as lifecycle refresh)
  const prompt = d.prepare('SELECT content FROM prompts WHERE id = ?').get(promptId);
  if (prompt && prompt.content && prompt.content.trim().length >= 15) {
    const content = prompt.content.trim();
    const contentWords = content.toLowerCase().split(/\s+/).filter(w => w.length > 4);

    if (contentWords.length >= 3) {
      const contentNorm = normalizePrompt(content);
      const ctmCwd = path.resolve(__dirname);
      const minMatchCount = Math.max(3, Math.ceil(contentWords.length * 0.5));
      const execs = d.prepare(`
        SELECT DISTINCT session_id, message_text, MAX(executed_at) as at, cwd
        FROM prompt_executions
        WHERE role = 'user'
        GROUP BY session_id, message_text
      `).all();

      for (const exec of execs) {
        if (seen.has(exec.session_id)) continue;
        if (exec.cwd && path.resolve(exec.cwd) === ctmCwd) continue; // skip CTM sessions

        const execLower = exec.message_text.toLowerCase();
        let matchCount = 0;
        for (const w of contentWords) {
          if (execLower.includes(w)) matchCount++;
        }

        const matched = matchCount >= minMatchCount;
        let substringMatched = false;
        if (!matched) {
          const execNorm = normalizePrompt(exec.message_text);
          substringMatched = execNorm.length >= 30 && (execNorm.includes(contentNorm.slice(0, 80)) || contentNorm.includes(execNorm.slice(0, 80)));
        }

        if (matched || substringMatched) {
          seen.add(exec.session_id);
          results.push({ session_id: exec.session_id, at: exec.at, source: 'matched' });
        }
      }
    }
  }

  // Sort by date desc and limit
  results.sort((a, b) => (b.at || '').localeCompare(a.at || ''));
  const limited = results.slice(0, limit);

  // Enrich with session titles from ctm_sessions (single source of truth)
  const titleStmt = d.prepare(`
    SELECT cs.title FROM ctm_sessions cs
    LEFT JOIN agent_sessions a ON a.ctm_session_id = cs.id
    WHERE cs.id = ? OR a.agent_session_id = ?
    ORDER BY cs.user_renamed DESC, cs.updated_at DESC LIMIT 1
  `);
  for (const r of limited) {
    const row = titleStmt.get(r.session_id, r.session_id);
    r.title = row?.title || null;
  }

  return limited;
}

function getPromptsForSession(sessionId) {
  const d = dbModule.getDb();
  // Get executions for this session
  const executions = d.prepare(`
    SELECT id, message_text, message_index, role, tool_uses, image_refs, source, executed_at
    FROM prompt_executions
    WHERE session_id = ? AND role = 'user'
    ORDER BY message_index ASC
  `).all(sessionId);

  // Also check if any library prompts were sent via prompt_usage
  const sentPrompts = d.prepare(`
    SELECT pu.prompt_id, p.title, pu.used_at
    FROM prompt_usage pu
    JOIN prompts p ON p.id = pu.prompt_id
    WHERE pu.session_id = ?
    ORDER BY pu.used_at ASC
  `).all(sessionId);

  return { executions, sentPrompts };
}

// ============================================================
// Outcome Tracking
// ============================================================

function setExecutionOutcome(executionId, outcome, notes) {
  const d = dbModule.getDb();
  d.prepare('UPDATE prompt_executions SET outcome = ?, outcome_notes = ? WHERE id = ?')
    .run(outcome, notes || null, executionId);
}

function getExecutionStats(promptId) {
  const d = dbModule.getDb();
  // Find executions that match this prompt's content
  const prompt = d.prepare('SELECT content FROM prompts WHERE id = ?').get(promptId);
  if (!prompt) return null;

  const norm = normalizePrompt(prompt.content);
  const executions = d.prepare(`
    SELECT outcome, count(*) as count
    FROM prompt_executions
    WHERE role = 'user' AND outcome IS NOT NULL
    GROUP BY outcome
  `).all();

  return { outcomes: executions };
}

// ============================================================
// Multi-Session Question Surfacing
// ============================================================

function getFrequentQuestions(limit = 20) {
  const d = dbModule.getDb();
  // Find user messages that appear in 2+ sessions (normalized)
  const rows = d.prepare(`
    SELECT message_text, COUNT(DISTINCT session_id) as session_count,
           GROUP_CONCAT(DISTINCT session_id) as sessions,
           MAX(executed_at) as last_used
    FROM prompt_executions
    WHERE role = 'user' AND length(message_text) > 30
    GROUP BY message_text
    HAVING session_count >= 2
    ORDER BY session_count DESC, last_used DESC
    LIMIT ?
  `).all(limit);

  // Also find normalized duplicates (same prompt, slightly different wording)
  const normalizedRows = d.prepare(`
    SELECT message_text, session_id, executed_at
    FROM prompt_executions
    WHERE role = 'user' AND length(message_text) > 30
    ORDER BY executed_at DESC
    LIMIT 3000
  `).all();

  const normMap = new Map();
  for (const row of normalizedRows) {
    const norm = normalizePrompt(row.message_text).slice(0, 150);
    if (norm.length < 20) continue;
    if (!normMap.has(norm)) normMap.set(norm, { text: row.message_text, sessions: new Set(), lastUsed: row.executed_at });
    normMap.get(norm).sessions.add(row.session_id);
    if (row.executed_at > normMap.get(norm).lastUsed) {
      normMap.get(norm).lastUsed = row.executed_at;
      normMap.get(norm).text = row.message_text;
    }
  }

  const normalizedDups = [];
  for (const [norm, data] of normMap) {
    if (data.sessions.size >= 2) {
      normalizedDups.push({
        text: data.text.slice(0, 200),
        sessionCount: data.sessions.size,
        sessions: [...data.sessions].slice(0, 5),
        lastUsed: data.lastUsed,
      });
    }
  }
  normalizedDups.sort((a, b) => b.sessionCount - a.sessionCount);

  // Merge exact and normalized, dedup
  const seen = new Set();
  const results = [];
  for (const r of rows) {
    const key = r.message_text.slice(0, 100).toLowerCase();
    if (seen.has(key)) continue;
    seen.add(key);
    results.push({
      text: r.message_text.slice(0, 200),
      sessionCount: r.session_count,
      sessions: r.sessions.split(',').slice(0, 5),
      lastUsed: r.last_used,
      type: 'exact',
    });
  }
  for (const r of normalizedDups) {
    const key = r.text.slice(0, 100).toLowerCase();
    if (seen.has(key)) continue;
    seen.add(key);
    results.push({ ...r, type: 'normalized' });
  }

  results.sort((a, b) => b.sessionCount - a.sessionCount);
  return results.slice(0, limit);
}

// Prompt quality assessment — purely local, no API calls
function assessPromptQuality(text) {
  if (!text || text.length < 5) return { quality: 'neutral', hint: '' };

  const words = text.split(/\s+/).filter(w => w.length > 1);
  const totalWords = words.length;
  if (totalWords < 3) return { quality: 'red', hint: 'Too short — add context about what you want done' };

  // Check for specificity signals
  const hasFilePath = /\/[\w.-]+/.test(text) || /\.(js|ts|py|css|html|json|md|sh|yml|yaml|go|rs|rb|java|c|cpp|h)\b/.test(text);
  const hasFunctionName = /\b[a-z][a-zA-Z0-9]*(?:_[a-z][a-zA-Z0-9]*)+\b/.test(text) || /\b[a-z]+[A-Z][a-zA-Z]+\b/.test(text);
  const hasConcreteVerb = /\b(add|remove|fix|refactor|rename|move|extract|replace|update|create|delete|implement|test|debug|optimize|migrate)\b/i.test(text);
  const hasCodeRef = /`[^`]+`/.test(text) || /\b(function|class|method|variable|component|module|endpoint|route|table|column|field)\b/i.test(text);
  const hasLineNumber = /line\s*\d+|L\d+/i.test(text);

  // Count specificity signals
  let signals = 0;
  if (hasFilePath) signals += 2;
  if (hasFunctionName) signals += 2;
  if (hasConcreteVerb) signals++;
  if (hasCodeRef) signals++;
  if (hasLineNumber) signals++;
  if (totalWords >= 10) signals++;

  // Vagueness signals
  const stopWords = new Set(['the','a','an','is','are','was','were','be','been','it','this','that','some','any','all','just','make','do','thing','stuff','something','everything','fix','change','update','help','please','can','you','i','my','we']);
  const contentWords = words.filter(w => !stopWords.has(w.toLowerCase()));
  const contentRatio = contentWords.length / totalWords;

  if (signals >= 3 || (signals >= 2 && contentRatio > 0.6)) {
    return { quality: 'green', hint: 'Prompt is specific and actionable' };
  } else if (signals >= 1 || (contentRatio > 0.5 && totalWords >= 8)) {
    let hint = '';
    if (!hasFilePath && !hasFunctionName) hint = 'Consider specifying which file or function to modify';
    else if (!hasConcreteVerb) hint = 'Consider using a specific action verb (add, fix, refactor, etc.)';
    else hint = 'Could be more specific about the expected outcome';
    return { quality: 'yellow', hint };
  } else {
    let hint = 'Too vague — specify the file, function, or concrete action';
    if (totalWords < 6) hint = 'Very short — describe what you want in more detail';
    return { quality: 'red', hint };
  }
}

// Ghost text completion — returns single best match for line prefix
function getGhostCompletion(prefix, context) {
  if (!prefix || prefix.length < 5) return null;
  const prefixLower = prefix.toLowerCase().trim();
  const index = getAutocompleteIndex();

  // Strategy 1: Find prompts starting with this prefix (history + library)
  const candidates = [];
  const seen = new Set();
  for (const entry of index) {
    if (entry.type === 'tool_context') continue;
    const fullText = (entry.fullText || entry.text || '');
    const fullLower = fullText.toLowerCase();
    const dedup = fullLower.slice(0, 150);
    if (seen.has(dedup)) continue;
    seen.add(dedup);

    // Check if the entry starts with or contains the prefix
    if (fullLower.startsWith(prefixLower)) {
      const completion = fullText.slice(prefix.length);
      if (completion.trim().length > 0) {
        candidates.push({ completion, confidence: 95, source: entry.type, text: fullText });
      }
    } else {
      // Check line-level: does any line in the entry start with prefix?
      const lines = fullLower.split('\n');
      for (const line of lines) {
        if (line.trimStart().startsWith(prefixLower.trimStart()) && line.trim().length > prefixLower.trim().length) {
          const origLine = fullText.split('\n').find(l => l.toLowerCase() === line);
          if (origLine) {
            const idx = origLine.toLowerCase().indexOf(prefixLower.trimStart());
            const completion = origLine.slice(idx + prefixLower.trimStart().length);
            if (completion.trim().length > 0) {
              candidates.push({ completion, confidence: 70, source: entry.type, text: origLine });
            }
          }
          break;
        }
      }
    }
  }

  // Boost candidates with higher historical effectiveness
  for (const c of candidates) {
    try {
      const eff = getEffectivenessForPrompt(c.text);
      if (eff && eff.sample_count >= 2) {
        c.confidence += Math.round(eff.avg_outcome * 15); // ±15 points based on outcome
      }
    } catch {}
  }

  // Strategy 2: BM25 ranking of candidates if multiple
  if (candidates.length === 0) return null;

  if (_bm25 && candidates.length > 1) {
    const docs = candidates.map(c => ({ ...c, text: c.text }));
    const ranked = _bm25.bm25Rerank(docs, prefix);
    // Combine BM25 score with prefix confidence
    for (const r of ranked) {
      r.finalScore = r.confidence + (r._bm25 || 0) * 5;
    }
    ranked.sort((a, b) => b.finalScore - a.finalScore);
    const best = ranked[0];
    if (best.confidence < 50) return null;
    return { completion: best.completion, confidence: best.confidence, source: best.source };
  }

  // Sort by confidence, return best
  candidates.sort((a, b) => b.confidence - a.confidence);
  const best = candidates[0];
  if (best.confidence < 50) return null;
  return { completion: best.completion, confidence: best.confidence, source: best.source };
}

// ============================================================
// Session Learning Pipeline — Implicit Outcome Detection
// ============================================================

// Infer outcomes for user prompts based on assistant responses and follow-up patterns
function inferOutcomes(sessionMessages) {
  if (!sessionMessages || sessionMessages.length < 2) return [];
  const outcomes = [];

  for (let i = 0; i < sessionMessages.length; i++) {
    const msg = sessionMessages[i];
    if (msg.role !== 'user') continue;

    let score = 0;
    let signals = [];
    const next = sessionMessages[i + 1]; // assistant response
    const followUp = sessionMessages[i + 2]; // next user message

    if (!next) continue; // No response yet

    const assistantText = (next.message_text || '').toLowerCase();
    const userText = (msg.message_text || '').toLowerCase();

    // Success signals from assistant response
    if (/\b(done|completed|i've (updated|modified|created|added|fixed|removed)|changes applied|successfully)\b/i.test(next.message_text || '')) {
      score += 0.3;
      signals.push('assistant_confirmed');
    }
    if (/\b(edit|write|bash|read)\b/i.test(assistantText) && assistantText.length > 200) {
      score += 0.2;
      signals.push('used_tools');
    }

    // Failure signals from assistant response
    if (/\b(sorry|apologize|i (can't|cannot)|error|failed|unable to)\b/i.test(next.message_text || '')) {
      score -= 0.3;
      signals.push('assistant_error');
    }

    // Follow-up signals
    if (followUp && followUp.role === 'user') {
      const followText = (followUp.message_text || '').toLowerCase();

      // Rephrased same request = failure
      if (userText.length > 20 && followText.length > 20) {
        const userWords = new Set(userText.split(/\s+/).filter(w => w.length > 3));
        const followWords = followText.split(/\s+/).filter(w => w.length > 3);
        const overlap = followWords.filter(w => userWords.has(w)).length;
        const ratio = overlap / Math.max(userWords.size, 1);
        if (ratio > 0.6) {
          score -= 0.3;
          signals.push('rephrased_request');
        }
      }

      // Explicit negative feedback
      if (/\b(no|wrong|try again|that's not|incorrect|undo|revert)\b/i.test(followText.slice(0, 100))) {
        score -= 0.4;
        signals.push('negative_feedback');
      }

      // Topic change = likely completed
      const userTopicWords = new Set(userText.split(/\s+/).filter(w => w.length > 4));
      const followTopicWords = followText.split(/\s+/).filter(w => w.length > 4);
      const topicOverlap = followTopicWords.filter(w => userTopicWords.has(w)).length;
      if (topicOverlap === 0 && followTopicWords.length > 3) {
        score += 0.2;
        signals.push('topic_changed');
      }
    } else if (!followUp) {
      // Last message in session — if assistant confirmed, likely success
      if (signals.includes('assistant_confirmed')) {
        score += 0.1;
        signals.push('session_ended_after_confirm');
      }
    }

    // Clamp to [-1.0, 1.0]
    score = Math.max(-1.0, Math.min(1.0, score));

    outcomes.push({
      messageIndex: msg.message_index,
      sessionId: msg.session_id,
      inferredOutcome: Math.round(score * 100) / 100,
      signals,
    });
  }

  return outcomes;
}

// ============================================================
// Prompt Effectiveness Table
// ============================================================

function ensureEffectivenessTable() {
  const d = dbModule.getDb();
  d.exec(`
    CREATE TABLE IF NOT EXISTS prompt_effectiveness (
      normalized_text TEXT PRIMARY KEY,
      avg_outcome REAL DEFAULT 0,
      sample_count INTEGER DEFAULT 0,
      best_variant TEXT,
      worst_variant TEXT,
      topics TEXT,
      updated_at TEXT DEFAULT (datetime('now'))
    )
  `);
  // Also ensure inferred_outcome column on prompt_executions
  try {
    d.exec('ALTER TABLE prompt_executions ADD COLUMN inferred_outcome REAL');
  } catch { /* column already exists */ }
}

function updatePromptEffectiveness() {
  const d = dbModule.getDb();
  ensureEffectivenessTable();

  // Get sessions with outcomes to process
  const sessions = d.prepare(`
    SELECT DISTINCT session_id FROM prompt_executions
    WHERE role = 'user' AND inferred_outcome IS NULL
    ORDER BY executed_at DESC LIMIT 50
  `).all();

  let processedCount = 0;

  for (const { session_id } of sessions) {
    const messages = d.prepare(`
      SELECT message_text, role, message_index, session_id, executed_at
      FROM prompt_executions
      WHERE session_id = ?
      ORDER BY message_index ASC
    `).all(session_id);

    const outcomes = inferOutcomes(messages);

    const updateStmt = d.prepare(`
      UPDATE prompt_executions SET inferred_outcome = ?
      WHERE session_id = ? AND message_index = ? AND role = 'user'
    `);

    for (const o of outcomes) {
      updateStmt.run(o.inferredOutcome, o.sessionId, o.messageIndex);
      processedCount++;
    }
  }

  // Now aggregate into prompt_effectiveness
  const aggregated = d.prepare(`
    SELECT
      message_text,
      AVG(inferred_outcome) as avg_outcome,
      COUNT(*) as sample_count,
      MAX(CASE WHEN inferred_outcome = (SELECT MAX(inferred_outcome) FROM prompt_executions pe2 WHERE pe2.message_text = pe.message_text) THEN message_text END) as best_variant,
      MIN(CASE WHEN inferred_outcome = (SELECT MIN(inferred_outcome) FROM prompt_executions pe2 WHERE pe2.message_text = pe.message_text) THEN message_text END) as worst_variant
    FROM prompt_executions pe
    WHERE role = 'user' AND inferred_outcome IS NOT NULL AND length(message_text) > 20
    GROUP BY message_text
    HAVING sample_count >= 1
    ORDER BY sample_count DESC
    LIMIT 500
  `).all();

  const upsert = d.prepare(`
    INSERT INTO prompt_effectiveness (normalized_text, avg_outcome, sample_count, best_variant, worst_variant, updated_at)
    VALUES (?, ?, ?, ?, ?, datetime('now'))
    ON CONFLICT(normalized_text) DO UPDATE SET
      avg_outcome = excluded.avg_outcome,
      sample_count = excluded.sample_count,
      best_variant = excluded.best_variant,
      worst_variant = excluded.worst_variant,
      updated_at = datetime('now')
  `);

  const txn = d.transaction(() => {
    for (const row of aggregated) {
      const norm = normalizePrompt(row.message_text);
      if (norm.length < 15) continue;
      upsert.run(
        norm.slice(0, 200),
        Math.round((row.avg_outcome || 0) * 100) / 100,
        row.sample_count,
        (row.best_variant || '').slice(0, 500),
        (row.worst_variant || '').slice(0, 500)
      );
    }
  });
  txn();

  return { processedOutcomes: processedCount, effectivenessEntries: aggregated.length };
}

// Quick lookup without table init (for use in loops after ensureEffectivenessTable already called)
function _getEffectivenessQuick(text) {
  if (!text || text.length < 20) return null;
  try {
    const norm = normalizePrompt(text);
    if (norm.length < 15) return null;
    return dbModule.getDb().prepare('SELECT * FROM prompt_effectiveness WHERE normalized_text = ?').get(norm.slice(0, 200)) || null;
  } catch { return null; }
}

function getEffectivenessForPrompt(text) {
  if (!text || text.length < 20) return null;
  try {
    ensureEffectivenessTable();
    return _getEffectivenessQuick(text);
  } catch { return null; }
}

module.exports = {
  // Harvester
  getHarvestState,
  runHarvest,
  getHarvestPreview,
  harvestSession,
  invalidateAutocompleteCache,

  // Autocomplete
  searchAutocomplete,
  findSimilarPrompts,
  getAutocompleteIndex,
  getGhostCompletion,

  // Patterns
  detectPatterns,
  normalizePrompt,

  // Quality
  assessPromptQuality,

  // Copilot
  getCopilotSuggestion,
  copilotChat,

  // Mapping
  getSessionsForPrompt,
  getPromptsForSession,

  // Lifecycle
  refreshLifecycleStatuses,
  runIncrementalLifecycleRefresh,

  // Outcomes
  setExecutionOutcome,
  getExecutionStats,

  // Multi-session questions
  getFrequentQuestions,

  // Session Learning
  inferOutcomes,
  ensureEffectivenessTable,
  updatePromptEffectiveness,
  getEffectivenessForPrompt,
};
