/*****************************************************************************\
|                                               ( )_  _                       |
|    _ _    _ __   _ _    __    ___ ___     _ _ | ,_)(_)  ___   ___     _     |
|   ( '_`\ ( '__)/'_` ) /'_ `\/' _ ` _ `\ /'_` )| |  | |/',__)/' v `\ /'_`\   |
|   | (_) )| |  ( (_| |( (_) || ( ) ( ) |( (_| || |_ | |\__, \| (˅) |( (_) )  |
|   | ,__/'(_)  `\__,_)`\__  |(_) (_) (_)`\__,_)`\__)(_)(____/(_) (_)`\___/'  |
|   | |                ( )_) |                                                |
|   (_)                 \___/'                                                |
|                                                                             |
| General Bots Copyright (c) Pragmatismo.io. All rights reserved.             |
| Licensed under the AGPL-3.0.                                                |
|                                                                             |
| According to our dual licensing model, this program can be used either      |
| under the terms of the GNU Affero General Public License, version 3,        |
| or under a proprietary license.                                             |
|                                                                             |
| The texts of the GNU Affero General Public License with an additional       |
| permission and of our proprietary license can be found at and               |
| in the LICENSE file you have received along with this program.              |
|                                                                             |
| This program is distributed in the hope that it will be useful,             |
| but WITHOUT ANY WARRANTY, without even the implied warranty of              |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                |
| GNU Affero General Public License for more details.                         |
|                                                                             |
| "General Bots" is a registered trademark of Pragmatismo.io.                 |
| The licensing of the program under the AGPLv3 does not imply a              |
| trademark license. Therefore any rights, title and interest in              |
| our trademarks remain entirely with us.                                     |
|                                                                             |
\*****************************************************************************/
/**
 * @fileoverview General Bots SSR support based on https://www.npmjs.com/package/ssr-for-bots.
 */
'use strict';
import puppeteer from 'puppeteer-extra';
import Fs from 'fs';
import urljoin from 'url-join';
// https://hackernoon.com/tips-and-tricks-for-web-scraping-with-puppeteer-ed391a63d952
// Dont download all resources, we just need the HTML
// Also, this is huge performance/response time boost
const blockedResourceTypes = ['image', 'media', 'font', 'texttrack', 'object', 'beacon', 'csp_report', 'imageset'];
// const whitelist = ["document", "script", "xhr", "fetch"];
const skippedResources = [
    'quantserve',
    'adzerk',
    'doubleclick',
    'adition',
    'exelator',
    'sharethrough',
    'cdn.api.twitter',
    'google-analytics',
    'googletagmanager',
    'google',
    'fontawesome',
    'facebook',
    'analytics',
    'optimizely',
    'clicktale',
    'mixpanel',
    'zedo',
    'clicksor',
    'tiqcdn'
];
const RENDER_CACHE = new Map();
async function createBrowser(profilePath) {
    let args = [
        '--check-for-update-interval=2592000',
        '--disable-accelerated-2d-canvas',
        '--disable-dev-shm-usage',
        '--disable-features=site-per-process',
        '--disable-gpu',
        '--no-first-run',
        '--no-default-browser-check'
    ];
    if (profilePath) {
        args.push(`--user-data-dir=${profilePath}`);
        const preferences = urljoin(profilePath, 'Default', 'Preferences');
        if (Fs.existsSync(preferences)) {
            const file = Fs.readFileSync(preferences, 'utf8');
            const data = JSON.parse(file);
            data['profile']['exit_type'] = 'none';
            Fs.writeFileSync(preferences, JSON.stringify(data));
        }
    }
    const browser = await puppeteer.launch({
        args: args,
        ignoreHTTPSErrors: true,
        headless: false,
        defaultViewport: null,
        ignoreDefaultArgs: ['--enable-automation', '--enable-blink-features=IdleDetection']
    });
    return browser;
}
async function recursiveFindInFrames(inputFrame, selector) {
    const frames = inputFrame.childFrames();
    const results = await Promise.all(frames.map(async (frame) => {
        const el = await frame.$(selector);
        if (el)
            return el;
        if (frame.childFrames().length > 0) {
            return await recursiveFindInFrames(frame, selector);
        }
        return null;
    }));
    return results.find(Boolean);
}
/**
 * https://developers.google.com/web/tools/puppeteer/articles/ssr#reuseinstance
 * @param {string} url URL to prerender.
 */
async function ssr(url, useCache, cacheRefreshRate) {
    if (RENDER_CACHE.has(url) && useCache) {
        const cached = RENDER_CACHE.get(url);
        if (Date.now() - cached.renderedAt > cacheRefreshRate && !(cacheRefreshRate <= 0)) {
            RENDER_CACHE.delete(url);
        }
        else {
            return {
                html: cached.html,
                status: 200
            };
        }
    }
    const browser = await createBrowser(null);
    const stylesheetContents = {};
    try {
        const page = await browser.newPage();
        await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36');
        await page.setRequestInterception(true);
        page.on('request', request => {
            const requestUrl = request
                .url()
                .split('?')[0]
                .split('#')[0];
            if (blockedResourceTypes.indexOf(request.resourceType()) !== -1 ||
                skippedResources.some(resource => requestUrl.indexOf(resource) !== -1)) {
                request.abort();
            }
            else {
                request.continue();
            }
        });
        page.on('response', async (resp) => {
            const responseUrl = resp.url();
            const sameOrigin = new URL(responseUrl).origin === new URL(url).origin;
            const isStylesheet = resp.request().resourceType() === 'stylesheet';
            if (sameOrigin && isStylesheet) {
                stylesheetContents[responseUrl] = await resp.text();
            }
        });
        const response = await page.goto(url, {
            timeout: 120000,
            waitUntil: 'networkidle0'
        });
        const sleep = ms => {
            return new Promise(resolve => {
                setTimeout(resolve, ms);
            });
        };
        await sleep(45000);
        // Inject <base> on page to relative resources load properly.
        await page.evaluate(url => {
            const base = document.createElement('base');
            base.href = url;
            // Add to top of head, before all other resources.
            document.head.prepend(base);
        }, url);
        // Remove scripts and html imports. They've already executed.
        await page.evaluate(() => {
            const elements = document.querySelectorAll('script, link[rel="import"]');
            elements.forEach(e => {
                e.remove();
            });
        });
        // Replace stylesheets in the page with their equivalent <style>.
        await page.$$eval('link[rel="stylesheet"]', (links, content) => {
            links.forEach((link) => {
                const cssText = content[link.href];
                if (cssText) {
                    const style = document.createElement('style');
                    style.textContent = cssText;
                    link.replaceWith(style);
                }
            });
        }, stylesheetContents);
        const html = await page.content();
        // Close the page we opened here (not the browser).
        await page.close();
        if (useCache) {
            RENDER_CACHE.set(url, { html, renderedAt: Date.now() });
        }
        return { html, status: response.status() };
    }
    catch (e) {
        const html = e.toString();
        console.warn({ message: `URL: ${url} Failed with message: ${html}` });
        return { html, status: 500 };
    }
    finally {
        await browser.close();
    }
}
function clearCache() {
    RENDER_CACHE.clear();
}
function ssrForBots(options = {
    prerender: [],
    exclude: [],
    useCache: true,
    cacheRefreshRate: 86400 // Seconds of which the cache will be kept alive, pass 0 or negative value for infinite lifespan
}) {
    let applyOptions = Object.assign({
        prerender: [],
        exclude: [],
        useCache: true,
        cacheRefreshRate: 86400 // Seconds of which the cache will be kept alive, pass 0 or negative value for infinite lifespan
    }, options);
    // Default user agents
    const prerenderArray = [
        'bot',
        'googlebot',
        'Chrome-Lighthouse',
        'DuckDuckBot',
        'ia_archiver',
        'bingbot',
        'yandex',
        'baiduspider',
        'Facebot',
        'facebookexternalhit',
        'facebookexternalhit/1.1',
        'twitterbot',
        'rogerbot',
        'linkedinbot',
        'embedly',
        'quora link preview',
        'showyoubot',
        'outbrain',
        'pinterest',
        'slackbot',
        'vkShare',
        'W3C_Validator'
    ];
    // default exclude array
    const excludeArray = ['.xml', '.ico', '.txt', '.json'];
    function ssrOnDemand(req, res, next) {
        Promise.resolve(() => {
            return true;
        })
            .then(async () => {
            const userAgent = req.headers['user-agent'] || '';
            const prerender = new RegExp([...prerenderArray, ...applyOptions.prerender].join('|').slice(0, -1), 'i').test(userAgent);
            const exclude = !new RegExp([...excludeArray, ...applyOptions.exclude].join('|').slice(0, -1)).test(req.originalUrl);
            if (req.originalUrl && prerender && exclude) {
                const { html, status } = await ssr(req.protocol + '://' + req.get('host') + req.originalUrl, applyOptions.useCache, applyOptions.cacheRefreshRate);
                return res.status(status).send(html);
            }
            else {
                return next();
            }
        })
            .catch(next);
    }
    return ssrOnDemand;
}
export { createBrowser, ssr, clearCache, ssrForBots };
//# sourceMappingURL=GBSSR.js.map