'use strict';

/**
 * Deduplication: finds semantically similar memories and archives shorter duplicates.
 * Uses embedding similarity when available, falls back to content overlap.
 */

const brain = require('../brain');
const { randomUUID: uuidv4 } = require('node:crypto');
let embeddings;
try { embeddings = require('../embeddings'); } catch { embeddings = null; }

/**
 * Find duplicate memory groups using embedding similarity.
 * @param {object} opts
 * @param {number} [opts.threshold=0.85] - Similarity threshold (0-1)
 * @param {string} [opts.source] - Filter by source
 * @param {string} [opts.domain] - Filter by domain
 * @param {boolean} [opts.dryRun=true] - If true, only report clusters without purging
 * @param {number} [opts.limit=1000] - Max memories to scan
 * @returns {object} { groups: Array<{ keeper, duplicates }>, stats }
 */
async function findDuplicates(opts = {}) {
  const { threshold = 0.85, source, domain, dryRun = true, limit = 1000 } = opts;
  const db = brain.getDb();

  // Build query for candidate memories
  const conditions = ['archived_at IS NULL', 'content IS NOT NULL', "length(content) > 20"];
  const params = [];
  if (source) { conditions.push('source = ?'); params.push(source); }
  if (domain) { conditions.push('domain = ?'); params.push(domain); }
  params.push(limit);

  const memories = db.prepare(`
    SELECT id, content, source, source_id, timestamp, importance FROM memories
    WHERE ${conditions.join(' AND ')}
    ORDER BY timestamp DESC LIMIT ?
  `).all(...params);

  if (memories.length < 2) return { groups: [], stats: { scanned: memories.length, groups: 0, duplicates: 0 } };

  const groups = [];
  const processed = new Set();

  // Use embeddings for similarity if available
  if (embeddings && embeddings.isAvailable()) {
    // Compute embeddings for all candidates
    const texts = memories.map(m => (m.content || '').slice(0, 512));
    const vectors = await embeddings.batchComputeEmbeddings(texts);

    if (vectors) {
      for (let i = 0; i < memories.length; i++) {
        if (processed.has(memories[i].id) || !vectors[i]) continue;

        const cluster = [];
        for (let j = i + 1; j < memories.length; j++) {
          if (processed.has(memories[j].id) || !vectors[j]) continue;

          const sim = cosineSimilarity(vectors[i], vectors[j]);
          if (sim >= threshold) {
            cluster.push({ memory: memories[j], similarity: sim });
            processed.add(memories[j].id);
          }
        }

        if (cluster.length > 0) {
          processed.add(memories[i].id);
          // Keeper = longest content
          const all = [{ memory: memories[i], similarity: 1.0 }, ...cluster];
          all.sort((a, b) => (b.memory.content || '').length - (a.memory.content || '').length);
          const keeper = all[0].memory;
          const duplicates = all.slice(1).map(d => ({ id: d.memory.id, similarity: d.similarity, length: (d.memory.content || '').length }));
          groups.push({ keeper: { id: keeper.id, length: (keeper.content || '').length }, duplicates });
        }
      }
    }
  } else {
    // Fallback: simple content overlap (Jaccard on words)
    for (let i = 0; i < memories.length; i++) {
      if (processed.has(memories[i].id)) continue;
      const wordsI = new Set((memories[i].content || '').toLowerCase().split(/\s+/));
      const cluster = [];

      for (let j = i + 1; j < memories.length; j++) {
        if (processed.has(memories[j].id)) continue;
        const wordsJ = new Set((memories[j].content || '').toLowerCase().split(/\s+/));
        const intersection = [...wordsI].filter(w => wordsJ.has(w)).length;
        const union = new Set([...wordsI, ...wordsJ]).size;
        const jaccard = union > 0 ? intersection / union : 0;

        if (jaccard >= threshold) {
          cluster.push({ memory: memories[j], similarity: jaccard });
          processed.add(memories[j].id);
        }
      }

      if (cluster.length > 0) {
        processed.add(memories[i].id);
        const all = [{ memory: memories[i], similarity: 1.0 }, ...cluster];
        all.sort((a, b) => (b.memory.content || '').length - (a.memory.content || '').length);
        const keeper = all[0].memory;
        const duplicates = all.slice(1).map(d => ({ id: d.memory.id, similarity: d.similarity, length: (d.memory.content || '').length }));
        groups.push({ keeper: { id: keeper.id, length: (keeper.content || '').length }, duplicates });
      }
    }
  }

  const totalDups = groups.reduce((sum, g) => sum + g.duplicates.length, 0);
  return { groups, stats: { scanned: memories.length, groups: groups.length, duplicates: totalDups } };
}

/**
 * Purge duplicates by archiving shorter memories in each group.
 * @param {Array} groups - Groups from findDuplicates
 * @returns {object} { archived: number }
 */
function purgeDuplicates(groups) {
  const db = brain.getDb();
  const now = new Date().toISOString();
  let archived = 0;

  const purge = db.transaction(() => {
    for (const group of groups) {
      const groupId = uuidv4();
      for (const dup of group.duplicates) {
        db.prepare("UPDATE memories SET archived_at = ?, dedup_group = ? WHERE id = ? AND archived_at IS NULL")
          .run(now, groupId, dup.id);
        archived++;
      }
    }
  });
  purge();

  return { archived };
}

/**
 * Run dedup once (for scheduled job).
 */
async function runOnce(opts = {}) {
  const result = await findDuplicates({ ...opts, dryRun: true });
  if (result.groups.length === 0) return { scanned: result.stats.scanned, archived: 0 };

  if (!opts.dryRun) {
    const purgeResult = purgeDuplicates(result.groups);
    console.log(`[dedup] Archived ${purgeResult.archived} duplicate memories from ${result.groups.length} groups`);
    return { scanned: result.stats.scanned, ...purgeResult };
  }

  return { scanned: result.stats.scanned, groups: result.groups.length, duplicates: result.stats.duplicates, dryRun: true };
}

function cosineSimilarity(a, b) {
  if (!a || !b || a.length !== b.length) return 0;
  let dot = 0, normA = 0, normB = 0;
  for (let i = 0; i < a.length; i++) {
    dot += a[i] * b[i];
    normA += a[i] * a[i];
    normB += b[i] * b[i];
  }
  const denom = Math.sqrt(normA) * Math.sqrt(normB);
  return denom > 0 ? dot / denom : 0;
}

module.exports = { findDuplicates, purgeDuplicates, runOnce, cosineSimilarity };
