import crypto from "crypto";
import { resolveSearchConfig, resolveSearchTokenLimits } from "./config.js";
function normalizeText(text) {
  return text.normalize("NFKD").replace(/[\u0300-\u036f]/g, "").replace(/[%_]/g, " ").toLowerCase();
}
function splitTokens(text, minLength) {
  return normalizeText(text).split(/[^a-z0-9]+/i).filter((token) => token.length >= minLength);
}
function appendExpandedToken(token, config, seen, tokens, limit) {
  const append = (candidate) => {
    if (seen.has(candidate)) return tokens.length < limit;
    seen.add(candidate);
    tokens.push(candidate);
    return tokens.length < limit;
  };
  if (!config.enablePartials) {
    append(token);
    return;
  }
  for (let length = config.minTokenLength; length <= token.length; length += 1) {
    if (!append(token.slice(0, length))) return;
  }
}
function hashToken(token, config) {
  const cfg = config ?? resolveSearchConfig();
  return crypto.createHash(cfg.hashAlgorithm).update(token).digest("hex");
}
function tokenizeText(text, config) {
  const cfg = config ?? resolveSearchConfig();
  const limits = resolveSearchTokenLimits(cfg);
  const boundedText = limits.maxFieldChars > 0 ? text.slice(0, limits.maxFieldChars) : text;
  const tokenLimit = limits.maxTokensPerField > 0 ? limits.maxTokensPerField : Number.POSITIVE_INFINITY;
  const seen = /* @__PURE__ */ new Set();
  const tokens = [];
  for (const token of splitTokens(boundedText, cfg.minTokenLength)) {
    if (tokens.length >= tokenLimit) break;
    appendExpandedToken(token, cfg, seen, tokens, tokenLimit);
  }
  const hashes = tokens.map((token) => hashToken(token, cfg));
  return { tokens, hashes };
}
export {
  hashToken,
  tokenizeText
};
//# sourceMappingURL=tokenize.js.map
