"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.normalizeWhitespace = normalizeWhitespace;
exports.tokenize = tokenize;
exports.chunkText = chunkText;
const DEFAULT_STOPWORDS = new Set([
    "a", "an", "the", "and", "or", "but", "if", "then", "else", "when", "while", "to", "of", "in", "on", "for", "with", "as",
    "is", "are", "was", "were", "be", "been", "being", "it", "this", "that", "these", "those", "by", "from", "at", "we", "you",
    "i", "they", "them", "their", "our", "us", "your", "he", "she", "his", "her", "not", "can", "could", "should", "would"
]);
function normalizeWhitespace(s) {
    return s.replace(/\s+/g, " ").trim();
}
function tokenize(input) {
    // lowercase, keep letters/numbers, split on everything else
    const tokens = input
        .toLowerCase()
        .replace(/[^a-z0-9]+/g, " ")
        .trim()
        .split(/\s+/)
        .filter(Boolean)
        .filter(t => t.length >= 2)
        .filter(t => !DEFAULT_STOPWORDS.has(t));
    return tokens;
}
function chunkText(text, chunkSize = 1200, overlap = 250) {
    // character-based chunking, simple and reliable
    const chunks = [];
    const clean = normalizeWhitespace(text);
    if (!clean)
        return chunks;
    let start = 0;
    while (start < clean.length) {
        const end = Math.min(clean.length, start + chunkSize);
        const chunk = clean.slice(start, end).trim();
        if (chunk.length > 0)
            chunks.push(chunk);
        if (end === clean.length)
            break;
        start = Math.max(0, end - overlap);
    }
    return chunks;
}
