# All files / composer/utils Scraper.js

0% Statements 0/18
0% Branches 0/9
0% Functions 0/5
0% Lines 0/17

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68

     /* globals Bugsnag */
    /**
     * Scrape urls to retrieve information about them.
     *
     * Returns a Promise that resolves with the JSON response from the appropriate
     * Scraper.urls entry as an object literal.
     *
     * Results are cached, and a same request done simultaneously will be
     * queued to return the same info as the already-running request.
     */

    import Request from '@bufferapp/buffer-js-request';

    class Scraper {
      static urls = new Map([
        ['local', 'https://scraper.local.buffer.com'],
        ['production', 'https://scraper.buffer.com'],
      ]);

      static pendingRequests = new Map(); // Stores pending scraper promises
      static cache = new Map(); // Stores results

      static encodeUrl(url) {
        // Ensure the url is encoded, but not double encoded
        try {
          return encodeURI(decodeURI(url));
        } catch (err) {
          return url;
        }
      }

      static scrape(url, environment = 'production') {
        // If the result is cached, return it right away
        if (this.cache.has(url)) return Promise.resolve(this.cache.get(url));

        // If a request about the same url is already running, return a
        // Promise that'll resolve with that request's data
        if (this.pendingRequests.has(url)) {
          return Promise.resolve(this.pendingRequests.get(url));
        }

        const encoded = this.encodeUrl(url);

        const request = Request.get(Scraper.urls.get(environment), { url: encoded })
          .then(response => {
            if (response.status >= 500 && environment === 'production') {
              Bugsnag.notify('ScraperError', 'Scraper returned 5xx error code', {
                url: encoded,
              });
            }
            return response;
          })
          .then((response) => response.json())
          .then((urlData) => {
            this.cache.set(url, urlData);
            this.pendingRequests.delete(url);

            return urlData;
          });

        this.pendingRequests.set(url, request);

        return request;
      }
    }

    export default Scraper;