'use strict';
// Phase 1a — embedding quantization recall eval (decision gate).
// Runs ONLY on a copy. Builds int8 + binary shadow vec tables from the existing
// float32 vectors, then measures recall@k by using stored vectors as query probes
// (this isolates *quantization distortion* — how much the quantized index reorders
// the same neighbors — without needing ollama to embed fresh queries).
//
// Usage: <node25> recall-eval.js [copyPath] [sampleN]
const path = require('path');
const WE = path.resolve(__dirname, '..', '..');
const Database = require(path.join(WE, 'node_modules', 'better-sqlite3'));
const vec = require(path.join(WE, 'node_modules', 'sqlite-vec'));

const COPY = process.argv[2] || path.join(process.env.HOME || process.cwd(), '.walle', 'opt-work', 'wall-e-brain.copy.db');
const SAMPLE = parseInt(process.argv[3] || '80', 10);
const KMAX = 30;

const db = new Database(COPY);
vec.load(db);
db.pragma('cache_size = -200000'); // 200MB page cache for the KNN scans

const SRC = 'embedding_vec_ollama';      // float[768]
const I8 = 'embedding_vec_ollama_i8';    // int8[768]
const BIT = 'embedding_vec_ollama_bit';  // bit[768]

function tableBytes(name) {
  // sum pgsize across the vec0 shadow tables for this logical table
  try {
    const r = db.prepare(
      `SELECT COALESCE(SUM(pgsize),0) AS b FROM dbstat WHERE name LIKE ? OR name LIKE ?`
    ).get(name + '%', 'sqlite_autoindex_' + name + '%');
    return r.b;
  } catch (e) { return -1; }
}

function buildQuant(target, type, expr) {
  const exists = db.prepare("SELECT 1 FROM sqlite_master WHERE name=?").get(target);
  if (exists) { db.exec(`DROP TABLE ${target}`); }
  db.exec(`CREATE VIRTUAL TABLE ${target} USING vec0(embedding ${type})`);
  const t0 = Date.now();
  try {
    db.exec(`INSERT INTO ${target}(rowid, embedding) SELECT rowid, ${expr} FROM ${SRC}`);
  } catch (e) {
    // fallback: row-by-row
    const sel = db.prepare(`SELECT rowid, embedding FROM ${SRC}`);
    const ins = db.prepare(`INSERT INTO ${target}(rowid, embedding) VALUES (?, ${expr.replace('embedding', '?')})`);
    const tx = db.transaction(() => { for (const r of sel.iterate()) ins.run(r.rowid, r.embedding); });
    tx();
  }
  return Date.now() - t0;
}

console.log(`# Embedding quantization recall eval`);
console.log(`copy: ${COPY}`);
const srcN = db.prepare(`SELECT COUNT(*) n FROM ${SRC}`).get().n;
console.log(`float32 vectors: ${srcN}, sample queries: ${SAMPLE}, k up to ${KMAX}\n`);

console.log('building int8 shadow table...');
const t8 = buildQuant(I8, 'int8[768]', "vec_quantize_int8(embedding,'unit')");
console.log(`  built in ${t8}ms`);
console.log('building binary(bit) shadow table...');
const tb = buildQuant(BIT, 'bit[768]', 'vec_quantize_binary(embedding)');
console.log(`  built in ${tb}ms\n`);

const bytesF = tableBytes(SRC), bytes8 = tableBytes(I8), bytesB = tableBytes(BIT);
const MB = b => (b / 1048576).toFixed(1);
console.log(`size  float32: ${MB(bytesF)} MB   int8: ${MB(bytes8)} MB (${(bytesF/bytes8).toFixed(1)}x)   bit: ${MB(bytesB)} MB (${(bytesF/bytesB).toFixed(1)}x)\n`);

// sample rowids
const allIds = db.prepare(`SELECT rowid FROM ${SRC}`).all().map(r => r.rowid);
function pick(n) {
  const out = []; const used = new Set(); let seed = 1234567;
  const rnd = () => { seed = (seed * 1103515245 + 12345) & 0x7fffffff; return seed / 0x7fffffff; };
  while (out.length < n && used.size < allIds.length) {
    const i = Math.floor(rnd() * allIds.length);
    if (!used.has(i)) { used.add(i); out.push(allIds[i]); }
  }
  return out;
}
const sample = pick(SAMPLE);

const getVec = db.prepare(`SELECT embedding FROM ${SRC} WHERE rowid=?`);
const knnF = db.prepare(`SELECT rowid FROM ${SRC} WHERE embedding MATCH ? AND k=? ORDER BY distance`);
const knn8 = db.prepare(`SELECT rowid FROM ${I8} WHERE embedding MATCH vec_quantize_int8(?,'unit') AND k=? ORDER BY distance`);
const knnB = db.prepare(`SELECT rowid FROM ${BIT} WHERE embedding MATCH vec_quantize_binary(?) AND k=? ORDER BY distance`);

function topk(stmt, qv, k, self) {
  const ids = stmt.all(qv, k + 1).map(r => r.rowid).filter(id => id !== self);
  return ids.slice(0, k);
}
function recallAt(gt, cand, k) {
  const g = new Set(gt.slice(0, k));
  let hit = 0; for (const id of cand.slice(0, k)) if (g.has(id)) hit++;
  return hit / k;
}

let sum8_10 = 0, sum8_30 = 0, sumB_10 = 0, sumB_30 = 0;
let tF = 0, t8t = 0, tBt = 0, worst8_10 = 1;
for (const rid of sample) {
  const qv = getVec.get(rid).embedding;
  let s = Date.now(); const gt = topk(knnF, qv, KMAX, rid); tF += Date.now() - s;
  s = Date.now(); const c8 = topk(knn8, qv, KMAX, rid); t8t += Date.now() - s;
  s = Date.now(); const cB = topk(knnB, qv, KMAX, rid); tBt += Date.now() - s;
  const r8_10 = recallAt(gt, c8, 10); const r8_30 = recallAt(gt, c8, 30);
  sum8_10 += r8_10; sum8_30 += r8_30;
  sumB_10 += recallAt(gt, cB, 10); sumB_30 += recallAt(gt, cB, 30);
  if (r8_10 < worst8_10) worst8_10 = r8_10;
}
const N = sample.length;
const pct = x => (100 * x / N).toFixed(1) + '%';
console.log('## Recall (quantized vs float32 ground truth), averaged over', N, 'real query probes');
console.log(`int8   recall@10: ${pct(sum8_10)}   recall@30: ${pct(sum8_30)}   worst@10: ${(100*worst8_10).toFixed(0)}%`);
console.log(`binary recall@10: ${pct(sumB_10)}   recall@30: ${pct(sumB_30)}   (binary normally needs a float rerank pass)`);
console.log();
console.log('## KNN latency (avg ms/query, exhaustive over', srcN, 'vectors)');
console.log(`float32: ${(tF/N).toFixed(1)}ms   int8: ${(t8t/N).toFixed(1)}ms   bit: ${(tBt/N).toFixed(1)}ms`);
db.close();
