"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.scrapUrl = void 0;
const got_1 = __importDefault(require("got"));
const jsdom_1 = require("jsdom");
const getUserAgent_1 = require("./getUserAgent");
const parseDom_1 = require("./parseDom");
async function scrapUrl(id, url) {
    try {
        let response = await (0, got_1.default)(url, {
            followRedirect: true,
            http2: true,
            retry: 3,
            headers: {
                "User-Agent": (0, getUserAgent_1.getUserAgent)(),
            },
        });
        if (/HTTP 30\d/.test(response.body)) {
            const [, redirectUrl] = response.body.match(/href="(.*)"/);
            try {
                response = await (0, got_1.default)(redirectUrl, {
                    followRedirect: true,
                    http2: true,
                    retry: 3,
                    headers: {
                        "User-Agent": (0, getUserAgent_1.getUserAgent)(),
                    },
                });
            }
            catch (error) {
                throw new Error(`Wrong redirectUrl: ${url} => ${redirectUrl}`);
            }
        }
        const dom = new jsdom_1.JSDOM(response.body, { url });
        const res = (0, parseDom_1.parseDom)(dom, id, url);
        return res;
    }
    catch (error) {
        let err;
        if (error instanceof got_1.default.ParseError) {
            err = new Error(`Parsing Error: ${error.message}`);
        }
        else if (error instanceof got_1.default.HTTPError) {
            err = new Error(`HTTP Error: ${error.response.statusCode} - ${url} - ${error.message}`);
            // Add a property to identify 403 errors specifically
            if (error.response.statusCode === 403) {
                err.isForbidden = true;
            }
        }
        else {
            err = new Error(error.message);
        }
        err.url = url;
        throw err;
    }
}
exports.scrapUrl = scrapUrl;
