import { logger } from "../utils/logger.js";
const BASE_URL = "https://arexplorer.zeroy.com/";
const DELAY_MS = 100;
const MAX_CONCURRENT = 5;
function delay(ms) {
    return new Promise((resolve) => setTimeout(resolve, ms));
}
/**
 * Fetch a single HTML page from the remote Doxygen mirror.
 */
async function fetchPage(url) {
    const response = await fetch(url, {
        headers: {
            "User-Agent": "enfusion-mcp-scraper/0.1.0 (https://github.com/enfusion-mcp)",
        },
    });
    if (!response.ok) {
        throw new Error(`HTTP ${response.status} for ${url}`);
    }
    return response.text();
}
/**
 * Fetch the annotated.html class list from the remote source.
 */
export async function fetchAnnotatedPage() {
    logger.info("Fetching class list from arexplorer.zeroy.com");
    return fetchPage(`${BASE_URL}annotated.html`);
}
/**
 * Fetch the hierarchy.html page from the remote source.
 */
export async function fetchHierarchyPage() {
    return fetchPage(`${BASE_URL}hierarchy.html`);
}
/**
 * Fetch multiple class pages with rate limiting.
 * Yields {filename, html} for each successfully fetched page.
 */
export async function* fetchClassPages(urls) {
    let pending = [];
    for (const { name, url } of urls) {
        const fullUrl = `${BASE_URL}${url}`;
        const promise = delay(DELAY_MS)
            .then(() => fetchPage(fullUrl))
            .then((html) => ({ filename: url, html }))
            .catch((err) => {
            logger.warn(`Failed to fetch ${name}: ${err}`);
            return null;
        });
        pending.push(promise);
        // Limit concurrency
        if (pending.length >= MAX_CONCURRENT) {
            const results = await Promise.all(pending);
            for (const result of results) {
                if (result)
                    yield result;
            }
            pending = [];
        }
    }
    // Flush remaining
    if (pending.length > 0) {
        const results = await Promise.all(pending);
        for (const result of results) {
            if (result)
                yield result;
        }
    }
}
//# sourceMappingURL=source-remote.js.map