// engine/spec-format/jaccard.ts — SPEC-724: Token-level Jaccard similarity.
const STOPWORDS = new Set(['the', 'a', 'an', 'de', 'la', 'el', 'y', 'o']);
/**
 * Tokenize text: split on whitespace and punctuation, lowercase, remove stopwords.
 */
function tokenize(text) {
    return new Set(text
        .toLowerCase()
        .split(/[\s\W]+/)
        .filter((t) => t.length > 0 && !STOPWORDS.has(t)));
}
/**
 * Compute the Jaccard similarity between two strings at the token level.
 * Returns a value in [0, 1]: 1 = identical token sets, 0 = fully disjoint.
 * Returns 1 when both strings are empty (vacuous equality).
 */
export function jaccard(a, b) {
    const tokensA = tokenize(a);
    const tokensB = tokenize(b);
    if (tokensA.size === 0 && tokensB.size === 0) {
        return 1;
    }
    const intersection = new Set();
    for (const t of tokensA) {
        if (tokensB.has(t)) {
            intersection.add(t);
        }
    }
    const union = new Set([...tokensA, ...tokensB]);
    return intersection.size / union.size;
}
//# sourceMappingURL=jaccard.js.map