all files / src/ queue-bull.js

97.3% Statements 36/37
90% Branches 9/10
100% Functions 5/5
100% Lines 29/29
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73                                                                                       
import createNormalQueue from 'bull';
import createPriorityQueue from 'bull/lib/priority-queue';
import redis from 'redis';
import bluebird from 'bluebird';
 
bluebird.promisifyAll(redis.RedisClient.prototype);
bluebird.promisifyAll(redis.Multi.prototype);
 
const REDIS_PORT = process.env.REDIS_PORT || 6379;
const REDIS_HOST = process.env.REDIS_HOST || '127.0.0.1';
 
const redisClient = redis.createClient({
  host: REDIS_HOST,
  port: REDIS_PORT,
});
 
const KEY_PREFIX = 'scraper.js-queue-bull-url-cache';
 
function getKey(queueItem) {
  return `${KEY_PREFIX}:(${queueItem.url})`;
}
 
export function removeCache(queueItem) {
  return redisClient.delAsync(getKey(queueItem));
}
 
export default function createBullQueue(name) {
  const createQueue = process.env.PRIORITY_QUEUE ? createPriorityQueue : createNormalQueue;
  const queue = createQueue(name, REDIS_PORT, REDIS_HOST);
 
  return {
    process(fn) {
      const callbackWrapper = (job, done) => fn(job.data, done);
      const promiseWrapper = job => fn(job.data);
 
      queue.process(fn.length === 2 ? callbackWrapper : promiseWrapper);
    },
    empty() {
      return queue.empty();
    },
    async add(queueItem) {
      const {
        expiry = 86400000,
        priority = 'normal',
        attempts = 2,
        backoff = 3600000,
      } = queueItem;
 
       // @todo was doing: removing options paramater and combining it with the queueItem
      const key = `${KEY_PREFIX}:(${queueItem.url})`;
      const saved = await redisClient.setnxAsync(key, 'true');
 
      if (saved === 1) {
        await queue.add(queueItem, {
          priority,
          attempts,
          backoff,
        });
        await redisClient.expireAsync(key, Math.ceil(expiry / 1000));
        return true;
      }
 
      return false;
    },
    count() {
      return queue.count();
    },
    close() {
      return queue.close();
    },
  };
}