import { Octokit } from "@octokit/rest";
import cheerio from "cheerio";
import fetch from "node-fetch";
import { URL } from "node:url";
const IGNORE_PATHS_ENDING_IN = [
    "favicon.ico",
    "robots.txt",
    ".rst.txt",
    "genindex",
    "py-modindex",
    "search.html",
    "search",
    "genindex.html",
    "changelog",
    "changelog.html",
];
const GITHUB_PATHS_TO_TRAVERSE = ["/blob/", "/tree/"];
async function crawlGithubRepo(baseUrl) {
    const octokit = new Octokit({
        auth: undefined,
    });
    const [_, owner, repo] = baseUrl.pathname.split("/");
    const dirContentsConfig = {
        owner: owner,
        repo: repo,
    };
    const tree = await octokit.request("GET /repos/{owner}/{repo}/git/trees/{tree_sha}", {
        owner,
        repo,
        tree_sha: "main",
        headers: {
            "X-GitHub-Api-Version": "2022-11-28",
        },
        recursive: "true",
    });
    const paths = tree.data.tree
        .filter((file) => file.type === "blob" && file.path?.endsWith(".md"))
        .map((file) => baseUrl.pathname + "/tree/main/" + file.path);
    return paths;
}
async function getLinksFromUrl(url, path) {
    const baseUrl = new URL(url);
    const location = new URL(path, url);
    let response;
    try {
        response = await fetch(location.toString());
    }
    catch (error) {
        if (error instanceof Error && error.message.includes("maximum redirect")) {
            console.error("Maximum redirect reached for: ", location.toString());
            return {
                html: "",
                links: [],
            };
        }
        console.error(error);
        return {
            html: "",
            links: [],
        };
    }
    const html = await response.text();
    let links = [];
    if (url.includes("github.com")) {
        return {
            html,
            links,
        };
    }
    const $ = cheerio.load(html);
    $("a").each((_, element) => {
        const href = $(element).attr("href");
        if (!href) {
            return;
        }
        const parsedUrl = new URL(href, url);
        if (parsedUrl.hostname === baseUrl.hostname
        // parsedUrl.pathname.startsWith(baseUrl.pathname)
        ) {
            links.push(parsedUrl.pathname);
        }
    });
    links = [...new Set(links)].filter((link) => {
        return (!link.includes("#") &&
            !IGNORE_PATHS_ENDING_IN.some((ending) => link.endsWith(ending)));
    });
    return {
        html,
        links,
    };
}
function splitUrl(url) {
    const baseUrl = `${url.protocol}//${url.hostname}${url.port ? ":" + url.port : ""}`;
    const basePath = url.pathname;
    return {
        baseUrl,
        basePath,
    };
}
export async function* crawlPage(url) {
    const { baseUrl, basePath } = splitUrl(url);
    let paths = [basePath];
    if (url.hostname === "github.com") {
        const githubLinks = await crawlGithubRepo(url);
        paths = [...paths, ...githubLinks];
    }
    let index = 0;
    while (index < paths.length) {
        const promises = paths
            .slice(index, index + 50)
            .map((path) => getLinksFromUrl(baseUrl, path));
        const results = await Promise.all(promises);
        for (const { html, links } of results) {
            if (html !== "") {
                yield {
                    url: url.toString(),
                    path: paths[index],
                    html: html,
                };
            }
            for (const link of links) {
                if (!paths.includes(link)) {
                    paths.push(link);
                }
            }
            index++;
        }
        paths = paths.filter((path) => results.some((result) => result.html !== "" && result.links.includes(path)));
    }
}
