'use strict';

/**
 * Embedding Search Quality Evaluation
 *
 * Compares search results: lexical-only vs hybrid (lexical + vector RRF).
 *
 * Usage:
 *   node scripts/eval-embeddings.js              # run all test queries
 *   node scripts/eval-embeddings.js "my query"   # run a single query
 *   node scripts/eval-embeddings.js --interactive # interactive mode
 */

const brain = require('../brain');
const embeddings = require('../embeddings');

brain.initDb();
embeddings.init();

const db = brain.getDb();

// ---- Search implementations ----

function lexicalSearchMemories(query, limit = 10) {
  const terms = extractTerms(query);
  if (!terms.length) return [];

  // Try FTS5 first
  const hasFts = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='memories_fts'").get();
  if (hasFts) {
    try {
      const ftsQuery = terms.join(' OR ');
      return db.prepare(`
        SELECT m.id, m.content, m.source, m.domain, m.topic, m.timestamp,
               f.relevance
        FROM (SELECT rowid, rank as relevance FROM memories_fts WHERE memories_fts MATCH ? ORDER BY rank LIMIT ?) f
        JOIN memories m ON m.rowid = f.rowid
      `).all(ftsQuery, limit);
    } catch {}
  }

  // LIKE fallback
  const conds = terms.map(() => 'content LIKE ?').join(' OR ');
  const params = terms.map(t => `%${t}%`);
  return db.prepare(`
    SELECT id, content, source, domain, topic, timestamp
    FROM memories WHERE (${conds})
    ORDER BY timestamp DESC LIMIT ?
  `).all(...params, limit);
}

async function hybridSearchMemories(query, limit = 10) {
  const lexical = lexicalSearchMemories(query, limit);

  try {
    const queryEmb = await embeddings.computeEmbedding(query);
    if (!queryEmb) return { lexical, hybrid: lexical, vector: [] };

    const vector = embeddings.searchSimilarMemories(queryEmb, limit);
    if (!vector.length) return { lexical, hybrid: lexical, vector: [] };

    const hybrid = embeddings.reciprocalRankFusion(lexical, vector, limit);
    return { lexical, hybrid, vector };
  } catch (err) {
    console.error('  Vector search failed:', err.message);
    return { lexical, hybrid: lexical, vector: [] };
  }
}

function lexicalSearchKnowledge(query, limit = 10) {
  const terms = extractTerms(query);
  if (!terms.length) return [];

  const conds = terms.map(() => '(subject LIKE ? OR predicate LIKE ? OR object LIKE ?)').join(' OR ');
  const params = [];
  for (const t of terms) params.push(`%${t}%`, `%${t}%`, `%${t}%`);

  return db.prepare(`
    SELECT id, category, subject, predicate, object, confidence
    FROM knowledge WHERE status = 'active' AND (${conds})
    ORDER BY confidence DESC LIMIT ?
  `).all(...params, limit);
}

async function hybridSearchKnowledge(query, limit = 10) {
  const lexical = lexicalSearchKnowledge(query, limit);

  try {
    const queryEmb = await embeddings.computeEmbedding(query);
    if (!queryEmb) return { lexical, hybrid: lexical, vector: [] };

    const vector = embeddings.searchSimilarKnowledge(queryEmb, limit);
    if (!vector.length) return { lexical, hybrid: lexical, vector: [] };

    const hybrid = embeddings.reciprocalRankFusion(lexical, vector, limit);
    return { lexical, hybrid, vector };
  } catch (err) {
    return { lexical, hybrid: lexical, vector: [] };
  }
}

// ---- Test queries ----
// Mix of exact-match-friendly, semantic, and fuzzy queries

const TEST_QUERIES = [
  // Semantic queries (embeddings should shine — meaning-based, not keyword match)
  { q: 'What makes me happy?', type: 'semantic', note: 'emotion-based, no keyword to match' },
  { q: 'Things I struggle with', type: 'semantic', note: 'abstract concept' },
  { q: 'My morning routine', type: 'semantic', note: 'behavioral pattern' },
  { q: 'How do I feel about my job?', type: 'semantic', note: 'sentiment analysis' },
  { q: 'What are my goals for this year?', type: 'semantic', note: 'future-oriented' },
  { q: 'People I spend the most time with', type: 'semantic', note: 'relationship inference' },

  // Factual/keyword queries (lexical should do well)
  { q: 'favorite restaurant', type: 'factual', note: 'simple keyword' },
  { q: 'birthday', type: 'factual', note: 'specific fact' },
  { q: 'coffee preference', type: 'factual', note: 'preference lookup' },

  // Paraphrase queries (same meaning, different words — embeddings should help)
  { q: 'places I traveled to recently', type: 'paraphrase', note: 'trips/vacations/visited' },
  { q: 'my health issues', type: 'paraphrase', note: 'medical/doctor/symptoms' },
  { q: 'stuff I bought recently', type: 'paraphrase', note: 'purchases/orders/shopping' },

  // Typo/fuzzy queries (embeddings are more tolerant)
  { q: 'exercize habits', type: 'fuzzy', note: 'typo in exercise' },

  // Cross-domain queries
  { q: 'work meetings this week', type: 'temporal', note: 'time-scoped' },
  { q: 'conversations about money', type: 'cross-domain', note: 'finance/budget/salary' },
];

// ---- Evaluation metrics ----

function overlap(listA, listB) {
  const idsA = new Set(listA.map(r => r.id));
  const idsB = new Set(listB.map(r => r.id));
  let common = 0;
  for (const id of idsA) if (idsB.has(id)) common++;
  return { common, onlyA: idsA.size - common, onlyB: idsB.size - common };
}

function rankShift(listA, listB) {
  // How much did ranking change? Lower = more similar ordering
  const rankA = new Map(listA.map((r, i) => [r.id, i]));
  const rankB = new Map(listB.map((r, i) => [r.id, i]));
  let totalShift = 0, compared = 0;
  for (const [id, posA] of rankA) {
    if (rankB.has(id)) {
      totalShift += Math.abs(posA - rankB.get(id));
      compared++;
    }
  }
  return compared > 0 ? totalShift / compared : -1;
}

function truncate(s, len = 80) {
  if (!s) return '(empty)';
  s = s.replace(/\n/g, ' ').trim();
  return s.length > len ? s.slice(0, len) + '...' : s;
}

// ---- Output ----

async function evaluateQuery(query, type, note) {
  const LIMIT = 10;
  const { lexical, hybrid, vector } = await hybridSearchMemories(query, LIMIT);
  const kResult = await hybridSearchKnowledge(query, LIMIT);

  const ov = overlap(lexical, hybrid);
  const rs = rankShift(lexical, hybrid);

  console.log(`\n${'='.repeat(90)}`);
  console.log(`Query: "${query}"  [${type}] ${note ? '(' + note + ')' : ''}`);
  console.log(`${'='.repeat(90)}`);
  console.log(`  Lexical: ${lexical.length} results | Vector: ${vector.length} results | Hybrid: ${hybrid.length} results`);
  console.log(`  Overlap: ${ov.common} shared, ${ov.onlyA} lexical-only, ${ov.onlyB} vector-only`);
  if (rs >= 0) console.log(`  Avg rank shift: ${rs.toFixed(1)} positions`);

  // Show unique vector finds (the value-add of embeddings)
  const lexIds = new Set(lexical.map(r => r.id));
  const vectorOnly = hybrid.filter(r => !lexIds.has(r.id));
  if (vectorOnly.length > 0) {
    console.log(`\n  [+] Embedding-only finds (not in lexical top ${LIMIT}):`);
    for (const r of vectorOnly.slice(0, 5)) {
      console.log(`      - [${r.source || '?'}] ${truncate(r.content, 100)}`);
      if (r.distance !== undefined) console.log(`        distance: ${r.distance.toFixed(4)}`);
    }
  }

  // Show top 3 from each
  console.log(`\n  Lexical top 3:`);
  for (const r of lexical.slice(0, 3)) {
    console.log(`    ${(r.relevance || 0).toFixed ? (r.relevance || 0).toFixed(2) : '-'} | [${r.source || '?'}] ${truncate(r.content, 90)}`);
  }
  console.log(`  Hybrid top 3:`);
  for (const r of hybrid.slice(0, 3)) {
    console.log(`    rrf:${(r._rrf_score || 0).toFixed(4)} | [${r.source || '?'}] ${truncate(r.content, 90)}`);
  }

  // Knowledge results
  if (kResult.lexical.length > 0 || kResult.vector?.length > 0) {
    console.log(`\n  Knowledge: ${kResult.lexical.length} lexical, ${kResult.vector?.length || 0} vector`);
    for (const k of (kResult.hybrid || kResult.lexical).slice(0, 3)) {
      console.log(`    ${k.subject} → ${k.predicate} → ${truncate(k.object, 60)}`);
    }
  }

  return {
    query, type,
    lexicalCount: lexical.length,
    vectorCount: vector.length,
    hybridCount: hybrid.length,
    overlap: ov.common,
    vectorOnlyFinds: ov.onlyB,
    avgRankShift: rs,
  };
}

async function runAll() {
  console.log('Embedding Search Quality Evaluation');
  console.log(`Memories: ${db.prepare('SELECT COUNT(*) as c FROM memories').get().c}`);
  console.log(`Embeddings: ${embeddings.getEmbeddingStats().total}`);
  console.log(`Knowledge: ${db.prepare("SELECT COUNT(*) as c FROM knowledge WHERE status='active'").get().c}`);
  console.log(`Model: ${embeddings.getModelConfig()?.name || 'none'}`);

  const results = [];
  for (const t of TEST_QUERIES) {
    results.push(await evaluateQuery(t.q, t.type, t.note));
  }

  // Summary
  console.log(`\n${'='.repeat(90)}`);
  console.log('SUMMARY');
  console.log(`${'='.repeat(90)}`);

  const withVectorFinds = results.filter(r => r.vectorOnlyFinds > 0);
  const avgVectorOnly = results.reduce((s, r) => s + r.vectorOnlyFinds, 0) / results.length;
  const avgOverlap = results.reduce((s, r) => s + r.overlap, 0) / results.length;
  const avgRankShift = results.filter(r => r.avgRankShift >= 0);

  console.log(`  Queries tested: ${results.length}`);
  console.log(`  Queries where embeddings found NEW results: ${withVectorFinds.length}/${results.length} (${(withVectorFinds.length / results.length * 100).toFixed(0)}%)`);
  console.log(`  Avg vector-only finds per query: ${avgVectorOnly.toFixed(1)}`);
  console.log(`  Avg lexical-hybrid overlap: ${avgOverlap.toFixed(1)}/${results[0]?.hybridCount || 10}`);
  if (avgRankShift.length) {
    const mrs = avgRankShift.reduce((s, r) => s + r.avgRankShift, 0) / avgRankShift.length;
    console.log(`  Avg rank shift (shared items): ${mrs.toFixed(1)} positions`);
  }

  // By query type
  const byType = {};
  for (const r of results) {
    if (!byType[r.type]) byType[r.type] = [];
    byType[r.type].push(r);
  }
  console.log('\n  By query type:');
  for (const [type, items] of Object.entries(byType)) {
    const vof = items.filter(r => r.vectorOnlyFinds > 0).length;
    const avgVO = items.reduce((s, r) => s + r.vectorOnlyFinds, 0) / items.length;
    console.log(`    ${type.padEnd(14)} | ${vof}/${items.length} improved | avg ${avgVO.toFixed(1)} new finds`);
  }
}

async function runSingle(query) {
  await evaluateQuery(query, 'custom', 'user query');
}

async function runInteractive() {
  const readline = require('readline');
  const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
  const ask = (q) => new Promise(r => rl.question(q, r));

  console.log('Interactive embedding eval. Type a query, or "quit" to exit.\n');
  while (true) {
    const q = await ask('query> ');
    if (!q || q === 'quit' || q === 'exit') break;
    await evaluateQuery(q, 'interactive', '');
  }
  rl.close();
}

// ---- Main ----
(async () => {
  const arg = process.argv[2];
  if (arg === '--interactive' || arg === '-i') {
    await runInteractive();
  } else if (arg) {
    await runSingle(arg);
  } else {
    await runAll();
  }
  process.exit(0);
})();

function extractTerms(msg) {
  if (!msg) return [];
  const stops = new Set([
    'the','and','for','are','but','not','you','all','can','had','her','was','one',
    'our','out','has','his','how','its','may','new','now','old','see','way','who',
    'did','get','let','say','she','too','use','what','when','where','which','with',
    'would','could','should','about','after','before','between','does','each','from',
    'have','just','like','make','many','more','most','much','must','name','only',
    'over','such','take','than','them','then','they','this','very','well','were',
    'will','your','tell','know','think','some','want','been','into','that',
  ]);
  return msg.toLowerCase().replace(/[^a-z0-9\s]/g, '').split(/\s+/)
    .filter(w => w.length >= 3 && !stops.has(w));
}
