import crypto from "crypto";
import { resolveSearchConfig } from "./config.js";
function normalizeText(text) {
  return text.normalize("NFKD").replace(/[\u0300-\u036f]/g, "").replace(/[%_]/g, " ").toLowerCase();
}
function splitTokens(text, minLength) {
  return normalizeText(text).split(/[^a-z0-9]+/i).filter((token) => token.length >= minLength);
}
function expandToken(token, config) {
  if (!config.enablePartials) return [token];
  const results = [];
  for (let i = config.minTokenLength; i <= token.length; i += 1) {
    results.push(token.slice(0, i));
  }
  return results;
}
function hashToken(token, config) {
  const cfg = config ?? resolveSearchConfig();
  return crypto.createHash(cfg.hashAlgorithm).update(token).digest("hex");
}
function tokenizeText(text, config) {
  const cfg = config ?? resolveSearchConfig();
  const baseTokens = splitTokens(text, cfg.minTokenLength);
  const expanded = baseTokens.flatMap((token) => expandToken(token, cfg));
  const unique = Array.from(new Set(expanded));
  const tokens = unique.filter((token) => token.length >= cfg.minTokenLength);
  const hashes = tokens.map((token) => hashToken(token, cfg));
  return { tokens, hashes };
}
export {
  hashToken,
  tokenizeText
};
//# sourceMappingURL=tokenize.js.map
