import { createHash } from "node:crypto";
import { getDatabase } from "../cache/db.js";
function splitIntoBlocks(markdown) {
  if (!markdown.trim()) return [];
  const lines = markdown.split("\n");
  const headingIndices = [];
  for (let i = 0; i < lines.length; i++) {
    const match = lines[i].match(/^(#{1,6})\s+/);
    if (match) {
      headingIndices.push({ level: match[1].length, lineIdx: i });
    }
  }
  if (headingIndices.length === 0) {
    return markdown.split(/\n\n+/).map((b) => b.trim()).filter(Boolean);
  }
  const blocks = [];
  for (let i = 0; i < headingIndices.length; i++) {
    const start = headingIndices[i].lineIdx;
    const end = i + 1 < headingIndices.length ? headingIndices[i + 1].lineIdx : lines.length;
    blocks.push(lines.slice(start, end).join("\n").trim());
  }
  return blocks.filter(Boolean);
}
function normalizeBlockText(text) {
  return text.toLowerCase().replace(/\s+/g, " ").trim();
}
function hashBlock(text) {
  return createHash("sha256").update(normalizeBlockText(text)).digest("hex");
}
const NAV_DEDUPE_THRESHOLD = 0.6;
const MAX_LEADING_LINES = 30;
const MAX_TRAILING_LINES = 20;
const MIN_CORPUS = 4;
function lineHash(line) {
  return createHash("sha1").update(line.trim().toLowerCase()).digest("hex");
}
function stripRepeatedNavigationLines(pages) {
  if (pages.length < MIN_CORPUS) return pages;
  const lineSets = pages.map((p) => p.markdown.split("\n"));
  const countLeading = /* @__PURE__ */ new Map();
  const countTrailing = /* @__PURE__ */ new Map();
  for (const lines of lineSets) {
    const seenL = /* @__PURE__ */ new Set();
    for (let i = 0; i < Math.min(MAX_LEADING_LINES, lines.length); i++) {
      const h = lineHash(lines[i]);
      if (!seenL.has(h)) {
        seenL.add(h);
        countLeading.set(h, (countLeading.get(h) ?? 0) + 1);
      }
    }
    const seenT = /* @__PURE__ */ new Set();
    for (let i = lines.length - 1; i >= Math.max(lines.length - MAX_TRAILING_LINES, 0); i--) {
      const h = lineHash(lines[i]);
      if (!seenT.has(h)) {
        seenT.add(h);
        countTrailing.set(h, (countTrailing.get(h) ?? 0) + 1);
      }
    }
  }
  const threshold = pages.length * NAV_DEDUPE_THRESHOLD;
  const navLeading = new Set([...countLeading].filter(([, c]) => c >= threshold).map(([h]) => h));
  const navTrailing = new Set([...countTrailing].filter(([, c]) => c >= threshold).map(([h]) => h));
  return pages.map((page, i) => {
    const lines = lineSets[i];
    let head = 0;
    while (head < lines.length && (lines[head].trim() === "" || navLeading.has(lineHash(lines[head])))) head++;
    let tail = lines.length;
    while (tail > head && (lines[tail - 1].trim() === "" || navTrailing.has(lineHash(lines[tail - 1])))) tail--;
    return { url: page.url, markdown: lines.slice(head, tail).join("\n") };
  });
}
function deduplicatePages(pages, domain) {
  if (pages.length <= 1) return pages.map((p) => ({ url: p.url, markdown: p.markdown }));
  const stripped = stripRepeatedNavigationLines(pages);
  const storedHashes = domain ? getStoredBoilerplate(domain) : [];
  const boilerplateHashes = new Set(storedHashes);
  const pageBlocks = stripped.map((page) => ({
    url: page.url,
    blocks: splitIntoBlocks(page.markdown)
  }));
  const hashPageCount = /* @__PURE__ */ new Map();
  for (const page of pageBlocks) {
    const seenHashes = /* @__PURE__ */ new Set();
    for (const block of page.blocks) {
      const h = hashBlock(block);
      if (!seenHashes.has(h)) {
        seenHashes.add(h);
        hashPageCount.set(h, (hashPageCount.get(h) ?? 0) + 1);
      }
    }
  }
  const threshold = pages.length / 2;
  for (const [hash, count] of hashPageCount) {
    if (count > threshold) {
      boilerplateHashes.add(hash);
    }
  }
  if (domain) {
    storeBoilerplate(domain, Array.from(boilerplateHashes));
  }
  return pageBlocks.map((page) => {
    const filtered = page.blocks.filter((block) => !boilerplateHashes.has(hashBlock(block)));
    return {
      url: page.url,
      markdown: filtered.join("\n\n")
    };
  });
}
function getStoredBoilerplate(domain) {
  const db = getDatabase();
  const rows = db.prepare("SELECT block_hash FROM domain_boilerplate WHERE domain = ?").all(domain);
  return rows.map((r) => r.block_hash);
}
function storeBoilerplate(domain, hashes) {
  const db = getDatabase();
  const del = db.prepare("DELETE FROM domain_boilerplate WHERE domain = ?");
  const insert = db.prepare(
    "INSERT OR IGNORE INTO domain_boilerplate (domain, block_hash, sample_text) VALUES (?, ?, ?)"
  );
  const tx = db.transaction((items) => {
    del.run(domain);
    for (const hash of items) {
      insert.run(domain, hash, null);
    }
  });
  tx(hashes);
}
export {
  deduplicatePages,
  getStoredBoilerplate,
  normalizeBlockText,
  splitIntoBlocks,
  storeBoilerplate,
  stripRepeatedNavigationLines
};
//# sourceMappingURL=dedup.js.map