'use strict';

/**
 * BM25 scoring for memory search re-ranking (G1).
 * Replaces raw LIKE matching with proper TF-IDF scoring.
 * Ported from MemPalace searcher.py (Okapi-BM25 with Lucene IDF).
 */

const K1 = 1.5;   // Term frequency saturation (typical range 1.2-2.0)
const B = 0.75;    // Length normalization (0=none, 1=full)
const TOKEN_RE = /[\w\u4e00-\u9fff]{2,}/gu; // Unicode-aware (includes CJK)

/**
 * Tokenize text into lowercase terms.
 */
function tokenize(text) {
  if (!text) return [];
  return (text.match(TOKEN_RE) || []).map(t => t.toLowerCase());
}

/**
 * Score documents against a query using Okapi-BM25.
 * @param {string[]} queryTokens - Tokenized query terms
 * @param {Array<{text: string}>} documents - Documents with text field
 * @returns {number[]} BM25 scores in document order
 */
function bm25Scores(queryTokens, documents) {
  if (!queryTokens.length || !documents.length) return documents.map(() => 0);

  const N = documents.length;
  // Pre-tokenize all documents
  const docTokens = documents.map(d => tokenize(d.text || d.content || ''));
  const avgDl = docTokens.reduce((s, t) => s + t.length, 0) / N || 1;

  // Compute IDF per query term (Lucene smoothed: always non-negative)
  const idf = {};
  for (const term of queryTokens) {
    const df = docTokens.filter(tokens => tokens.includes(term)).length;
    idf[term] = Math.log((N - df + 0.5) / (df + 0.5) + 1);
  }

  return docTokens.map(tokens => {
    let score = 0;
    const dl = tokens.length;
    for (const term of queryTokens) {
      const freq = tokens.filter(t => t === term).length;
      if (freq === 0) continue;
      score += (idf[term] || 0) * freq * (K1 + 1)
        / (freq + K1 * (1 - B + B * dl / avgDl));
    }
    return score;
  });
}

/**
 * Merge vector similarity and BM25 keyword scores with min-max normalization.
 * @param {Array} results - Results with optional vectorSim field
 * @param {string[]} queryTokens - Tokenized query terms
 * @param {number} vectorWeight - Weight for vector similarity (default 0.6)
 * @param {number} bm25Weight - Weight for BM25 (default 0.4)
 * @returns {Array} Results sorted by hybrid score
 */
function hybridRank(results, queryTokens, vectorWeight = 0.6, bm25Weight = 0.4) {
  if (!results.length) return results;

  const scores = bm25Scores(queryTokens, results);
  const maxBm25 = Math.max(...scores, 0.001);
  const minBm25 = Math.min(...scores);
  const range = maxBm25 - minBm25 || 1;

  return results.map((r, i) => ({
    ...r,
    _bm25: scores[i],
    _bm25Norm: (scores[i] - minBm25) / range,
    _hybridScore: vectorWeight * (r._vectorSim || 0) + bm25Weight * ((scores[i] - minBm25) / range),
  })).sort((a, b) => b._hybridScore - a._hybridScore);
}

/**
 * Re-rank a set of search results using BM25 (no vector component).
 * For use when embeddings aren't available.
 * @param {Array} results - Results with content/text field
 * @param {string} query - Raw query string
 * @returns {Array} Results sorted by BM25 score
 */
function bm25Rerank(results, query) {
  if (!results.length || !query) return results;
  const queryTokens = tokenize(query);
  if (!queryTokens.length) return results;

  const scores = bm25Scores(queryTokens, results);
  return results
    .map((r, i) => ({ ...r, _bm25: scores[i] }))
    .sort((a, b) => b._bm25 - a._bm25);
}

module.exports = { tokenize, bm25Scores, hybridRank, bm25Rerank };
