"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.loadWebText = loadWebText;
const jsdom_1 = require("jsdom");
const readability_1 = require("@mozilla/readability");
const DEFAULTS = {
    timeoutMs: 20000,
    minTextLength: 400,
    userAgent: "Mozilla/5.0 (compatible; MinimalRetriever/1.0; +https://example.com)",
};
// --- Public API ---
async function loadWebText(url, opts = {}) {
    var _a, _b, _c;
    const o = Object.assign(Object.assign({}, DEFAULTS), opts);
    const html = await fetchHtml(url, o);
    const dom = new jsdom_1.JSDOM(html, { url });
    const doc = dom.window.document;
    // Try Readability on an untouched clone first
    const readableText = tryReadability(doc);
    const cleanedReadable = deJunk(readableText);
    if (cleanedReadable.length >= o.minTextLength) {
        return normalizeWhitespace(cleanedReadable);
    }
    // Only then do aggressive fallback cleanup
    preStripDom(doc);
    const mainEl = pickMain(doc);
    const fallbackText = deJunk((_c = (_a = mainEl === null || mainEl === void 0 ? void 0 : mainEl.textContent) !== null && _a !== void 0 ? _a : (_b = doc.body) === null || _b === void 0 ? void 0 : _b.textContent) !== null && _c !== void 0 ? _c : "");
    return normalizeWhitespace(fallbackText);
}
// --- Fetch ---
async function fetchHtml(url, o) {
    const controller = new AbortController();
    const t = setTimeout(() => controller.abort(), o.timeoutMs);
    try {
        const res = await fetch(url, {
            signal: controller.signal,
            headers: {
                "User-Agent": o.userAgent,
                Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                "Accept-Language": "en-US,en;q=0.9",
                "Cache-Control": "no-cache",
                Pragma: "no-cache",
            },
        });
        if (!res.ok)
            throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
        const contentType = res.headers.get("content-type") || "";
        if (!contentType.includes("text/html") && !contentType.includes("application/xhtml+xml")) {
            // Some sites mislabel; still attempt, but avoid indexing binary junk.
            // If it’s clearly not text-ish, bail.
            if (contentType && !contentType.includes("text/")) {
                throw new Error(`Unsupported content-type for ${url}: ${contentType}`);
            }
        }
        return await res.text();
    }
    finally {
        clearTimeout(t);
    }
}
// --- DOM cleanup ---
function preStripDom(doc) {
    doc.querySelectorAll("script, style, noscript, svg, canvas, iframe, form, input, button, select, textarea")
        .forEach(n => n.remove());
    doc.querySelectorAll("nav, footer, aside, [role='navigation'], [role='contentinfo']")
        .forEach(n => n.remove());
    const junkAttrMatchers = [
        /cookie/i,
        /consent/i,
        /gdpr/i,
        /newsletter/i,
        /subscribe/i,
        /promo/i,
        /modal/i,
        /popup/i,
        /paywall/i,
        /overlay/i,
    ];
    const safeContentHints = [
        /article/i,
        /content/i,
        /post/i,
        /entry/i,
        /main/i,
        /body/i,
        /markdown/i,
    ];
    const all = Array.from(doc.querySelectorAll("*"));
    for (const el of all) {
        const id = el.id;
        const cls = el.className;
        const role = el.getAttribute("role") || "";
        const aria = el.getAttribute("aria-label") || "";
        const data = [
            el.getAttribute("data-testid") || "",
            el.getAttribute("data-test") || "",
            el.getAttribute("data-qa") || "",
        ].join(" ");
        const hay = `${id} ${cls} ${role} ${aria} ${data}`;
        if (safeContentHints.some(r => r.test(hay)))
            continue;
        if (junkAttrMatchers.some(r => r.test(hay))) {
            el.remove();
        }
    }
}
// --- Readability extraction ---
function tryReadability(doc) {
    var _a;
    try {
        const cloned = doc.cloneNode(true);
        const reader = new readability_1.Readability(cloned);
        const article = reader.parse();
        return (_a = article === null || article === void 0 ? void 0 : article.textContent) !== null && _a !== void 0 ? _a : "";
    }
    catch (err) {
        console.error("Readability failed:", err);
        return "";
    }
}
// --- Fallback selector logic ---
function pickMain(doc) {
    var _a, _b, _c, _d;
    // Try likely content containers first
    const selectors = [
        "article",
        "main",
        "[role='main']",
        ".markdown",
        ".content",
        ".post-content",
        ".article-content",
        ".entry-content",
        ".postBody",
        ".post-body",
        "#content",
        "#main",
        "#article",
    ];
    let best = null;
    let bestScore = 0;
    for (const s of selectors) {
        const el = doc.querySelector(s);
        if (!el)
            continue;
        const txt = ((_a = el.textContent) !== null && _a !== void 0 ? _a : "").trim();
        const score = scoreText(txt);
        if (score > bestScore) {
            best = el;
            bestScore = score;
        }
    }
    // If none matched well, pick the “largest meaningful” block from body
    if (!best) {
        const candidates = Array.from((_c = (_b = doc.body) === null || _b === void 0 ? void 0 : _b.querySelectorAll("section, div, article, main")) !== null && _c !== void 0 ? _c : []);
        for (const el of candidates) {
            const txt = ((_d = el.textContent) !== null && _d !== void 0 ? _d : "").trim();
            const score = scoreText(txt);
            if (score > bestScore) {
                best = el;
                bestScore = score;
            }
        }
    }
    return best;
}
function scoreText(t) {
    // Reward length but penalize obvious nav/boilerplate-ish density patterns
    const len = t.length;
    if (len < 200)
        return 0;
    const lines = t.split(/\n+/).map(x => x.trim()).filter(Boolean);
    const shortLines = lines.filter(l => l.length <= 40).length;
    const shortRatio = lines.length ? shortLines / lines.length : 1;
    // Too many tiny lines often means menus
    const penalty = shortRatio > 0.6 ? 0.4 : 1.0;
    return len * penalty;
}
// --- Text cleanup ---
function normalizeWhitespace(s) {
    return s.replace(/\r/g, "").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").replace(/[ \t]+/g, " ").trim();
}
function deJunk(text) {
    var _a;
    if (!text)
        return "";
    // Split into lines, then filter hard
    const rawLines = text
        .replace(/\r/g, "")
        .split("\n")
        .map(l => l.trim())
        .filter(Boolean);
    // Patterns that scream “not content”
    const badLine = [
        /cookie/i,
        /consent/i,
        /gdpr/i,
        /accept all/i,
        /reject all/i,
        /manage (your )?preferences/i,
        /privacy policy/i,
        /terms of (service|use)/i,
        /do not sell/i,
        /subscribe/i,
        /newsletter/i,
        /sign up/i,
        /log in/i,
        /register/i,
        /advertis/i,
        /sponsored/i,
        /all rights reserved/i,
        /©\s?\d{4}/i,
        /share (this|on)/i,
        /follow us/i,
        /related (posts|articles)/i,
        /recommended/i,
        /read more/i,
        /^menu$/i,
        /^home$/i,
        /^search$/i,
        /^skip to/i,
    ];
    // Remove near-duplicates and ultra-short boilerplate
    const seen = new Set();
    const kept = [];
    for (const line of rawLines) {
        const l = line.replace(/\s+/g, " ").trim();
        if (!l)
            continue;
        // Drop very short lines unless they look like real sentences
        if (l.length < 25 && !/[.!?]$/.test(l))
            continue;
        // Drop huge “lines” that are usually minified junk
        if (l.length > 5000)
            continue;
        if (badLine.some(r => r.test(l)))
            continue;
        const key = l.toLowerCase();
        if (seen.has(key))
            continue;
        seen.add(key);
        // Drop nav-like lines with tons of separators
        const sepCount = ((_a = l.match(/[|•»›·]/g)) !== null && _a !== void 0 ? _a : []).length;
        if (sepCount >= 4 && l.length < 200)
            continue;
        kept.push(l);
    }
    // One more pass: collapse repeated “sections” that appear multiple times
    return kept.join("\n");
}
