"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.scrap = exports.fetchFeed = void 0;
const fs_1 = __importDefault(require("fs"));
const p_limit_1 = __importDefault(require("p-limit"));
const path_1 = __importDefault(require("path"));
const scrapUrl_1 = require("./scrapUrl");
const FEED_URL = "https://travail-emploi.gouv.fr/?page=oseo_json";
const limit = (0, p_limit_1.default)(10);
async function fetchFeed() {
    const localJsonData = fs_1.default.readFileSync(path_1.default.join(__dirname, "../../local.data.json"), "utf8");
    return JSON.parse(localJsonData).fiches;
}
exports.fetchFeed = fetchFeed;
async function scrap(urls) {
    const inputs = urls.map(({ id, url }) => limit(() => (0, scrapUrl_1.scrapUrl)(id, url)));
    const results = await Promise.allSettled(inputs);
    const failedPromise = results.filter(({ status }) => status === "rejected");
    // Separate 403 errors from other errors
    const forbiddenErrors = failedPromise.filter(({ reason }) => reason.isForbidden);
    const otherErrors = failedPromise.filter(({ reason }) => !reason.isForbidden);
    // Log 403 errors as warnings and save them to a file for GitHub Actions
    if (forbiddenErrors.length > 0) {
        const forbiddenUrls = forbiddenErrors.map(({ reason }) => reason.url);
        console.warn("WARNING: The following pages returned 403 Forbidden and were skipped:", forbiddenUrls);
        // Write forbidden URLs to a file for GitHub Actions to use
        fs_1.default.writeFileSync(path_1.default.join(__dirname, "../../forbidden-urls.json"), JSON.stringify({
            urls: forbiddenUrls,
            timestamp: new Date().toISOString(),
            count: forbiddenUrls.length,
        }, null, 2));
    }
    // Only fail if there are non-403 errors
    if (otherErrors.length > 0) {
        console.error("scrap fail", otherErrors.map(({ reason }) => reason));
        throw new Error("Error - fetching pages fail. Some pages are missing");
    }
    const resolvedPromise = results.flatMap(({ status, value }) => status === "fulfilled" ? [value] : []);
    // ensure we not have duplicate url
    let hasDuplicate = false;
    for (const { pubId, url } of resolvedPromise) {
        const count = resolvedPromise.filter((fiche) => fiche.pubId === pubId && pubId !== undefined).length;
        if (count > 1) {
            hasDuplicate = true;
            console.error(`[error] la fiche ${url} est présente ${count} fois. Veuillez supprimer le doublon du datafiller`);
        }
    }
    if (hasDuplicate) {
        throw new Error(`[error] fiches en doublons. Veuillez supprimer les doublons du datafiller`);
    }
    return resolvedPromise;
}
exports.scrap = scrap;
if (module === require.main) {
    const t0 = Date.now();
    fetchFeed(FEED_URL)
        .then(scrap)
        .then((fiches) => {
        console.log(`done in ${Math.round((Date.now() - t0) / 1000)} sec`);
        const dataFilePath = path_1.default.join(__dirname, "..", "..", "data", "fiches-travail.json");
        fs_1.default.mkdirSync(path_1.default.dirname(dataFilePath), { recursive: true });
        fs_1.default.writeFileSync(dataFilePath, JSON.stringify(fiches, null, 2));
    })
        .catch((error) => {
        console.error(error);
        console.error(`fail in ${Math.round((Date.now() - t0) / 1000)} sec`);
        process.exit(1);
    });
}
