// engine/vector-store/tfidf.ts — TF-IDF embedding engine.
// SPEC-075 AC-01: Tokenize, compute TF-IDF, generate dense vectors.
import { normalizeVector } from './similarity.js';
const DEFAULT_STOP_WORDS = new Set([
    // English
    'the',
    'a',
    'an',
    'is',
    'are',
    'was',
    'were',
    'be',
    'been',
    'being',
    'have',
    'has',
    'had',
    'do',
    'does',
    'did',
    'will',
    'would',
    'could',
    'should',
    'may',
    'might',
    'shall',
    'can',
    'need',
    'dare',
    'ought',
    'to',
    'of',
    'in',
    'for',
    'on',
    'with',
    'at',
    'by',
    'from',
    'as',
    'into',
    'through',
    'during',
    'before',
    'after',
    'above',
    'below',
    'between',
    'out',
    'off',
    'over',
    'under',
    'again',
    'further',
    'then',
    'once',
    'here',
    'there',
    'when',
    'where',
    'why',
    'how',
    'all',
    'each',
    'every',
    'both',
    'few',
    'more',
    'most',
    'other',
    'some',
    'such',
    'no',
    'nor',
    'not',
    'only',
    'own',
    'same',
    'so',
    'than',
    'too',
    'very',
    'just',
    'because',
    'but',
    'and',
    'or',
    'if',
    'while',
    'this',
    'that',
    'these',
    'those',
    'it',
    'its',
    'i',
    'me',
    'my',
    'we',
    'our',
    'you',
    'your',
    'he',
    'him',
    'his',
    'she',
    'her',
    'they',
    'them',
    'their',
    'what',
    'which',
    'who',
    'whom',
    'about',
    'up',
    // Spanish
    'el',
    'la',
    'los',
    'las',
    'un',
    'una',
    'unos',
    'unas',
    'es',
    'son',
    'de',
    'del',
    'en',
    'con',
    'por',
    'para',
    'se',
    'que',
    'como',
    'pero',
    'si',
    'su',
    'al',
    'lo',
    'le',
    'les',
    'nos',
    'ya',
    'hay',
    'o',
    'y',
]);
/** Tokenize text: split on whitespace/punctuation, split camelCase, lowercase, remove stop words. */
export function tokenize(text, stopWords = DEFAULT_STOP_WORDS) {
    // Split camelCase: "camelCase" -> "camel Case" -> "camel case"
    const expanded = text.replace(/([a-z])([A-Z])/g, '$1 $2');
    // Split on non-alphanumeric
    const raw = expanded
        .toLowerCase()
        .split(/[^a-z0-9]+/)
        .filter(Boolean);
    return raw.filter((t) => t.length > 1 && !stopWords.has(t));
}
/** TF-IDF engine: stateful, maintains vocabulary (IDF) across documents. */
export class TFIDFEngine {
    config;
    docFreq = new Map();
    docCount = 0;
    vocabulary = [];
    vocabDirty = true;
    constructor(config) {
        this.config = {
            maxDimensions: config?.maxDimensions ?? 256,
            minDocFrequency: config?.minDocFrequency ?? 1,
            stopWords: config?.stopWords ?? DEFAULT_STOP_WORDS,
        };
    }
    /** Add a document to the corpus (updates IDF). */
    addDocument(text) {
        const tokens = tokenize(text, this.config.stopWords);
        const seen = new Set(tokens);
        for (const term of seen) {
            this.docFreq.set(term, (this.docFreq.get(term) ?? 0) + 1);
        }
        this.docCount++;
        this.vocabDirty = true;
    }
    /** Rebuild vocabulary: top-K most discriminating terms by IDF. */
    rebuildVocabulary() {
        if (!this.vocabDirty) {
            return;
        }
        const entries = [...this.docFreq.entries()]
            .filter(([, df]) => df >= this.config.minDocFrequency)
            .map(([term, df]) => ({
            term,
            idf: Math.log((this.docCount + 1) / (df + 1)) + 1,
        }))
            .sort((a, b) => b.idf - a.idf);
        this.vocabulary = entries.slice(0, this.config.maxDimensions).map((e) => e.term);
        this.vocabDirty = false;
    }
    /** Generate a normalized TF-IDF embedding for the given text. */
    embed(text) {
        this.rebuildVocabulary();
        const tokens = tokenize(text, this.config.stopWords);
        if (tokens.length === 0) {
            return new Array(this.vocabulary.length).fill(0);
        }
        // Compute TF
        const tf = new Map();
        for (const t of tokens) {
            tf.set(t, (tf.get(t) ?? 0) + 1);
        }
        // Build vector
        const vector = this.vocabulary.map((term) => {
            /* v8 ignore start -- ?? fallbacks: term always in tf/docFreq from vocabulary */
            const termFreq = (tf.get(term) ?? 0) / tokens.length;
            const df = this.docFreq.get(term) ?? 0;
            /* v8 ignore stop */
            const idf = Math.log((this.docCount + 1) / (df + 1)) + 1;
            return termFreq * idf;
        });
        return normalizeVector(vector);
    }
    /** Batch embed multiple texts. */
    batchEmbed(texts) {
        return texts.map((t) => this.embed(t));
    }
    /** Get current vocabulary size. */
    getVocabularySize() {
        this.rebuildVocabulary();
        return this.vocabulary.length;
    }
    /** Get document count in corpus. */
    getDocumentCount() {
        return this.docCount;
    }
    /** Export state for serialization. */
    exportState() {
        return {
            docFreq: [...this.docFreq.entries()],
            docCount: this.docCount,
        };
    }
    /** Import state from serialized form. */
    importState(state) {
        this.docFreq.clear();
        for (const [term, freq] of state.docFreq) {
            this.docFreq.set(term, freq);
        }
        this.docCount = state.docCount;
        this.vocabDirty = true;
    }
}
//# sourceMappingURL=tfidf.js.map