qryq

stop RESTing, start using query queues

/ˈkwərik/

Brendan Graetz

Brendan Graetz

@bguiz

bguiz.com


Shine Technologies

Shine Technologies

@realshinetech

shinetech.com


In one sentence

qry is a NodeJs library that allows one to express a series of API queries and define the dependencies between them. These queries may be executed in parallel, in sequence, or in a directed acyclic graph.


The Query Queue

  
[
  {"id":"q1","depends":[],"api":"add","qry":{"a":1,"b":9}},
  {"id":"q2","depends":[],"api":"add","qry":{"a":99,"b":1}},
  {"id":"q3","depends":["q2","q1"],"api":"multiply","qry":{"a":"#{q1}","b":"#{q2}"}},
  {"id":"q4","depends":["q3"],"api":"multiply","qry":{"a":"#{q3}","b":5}}
]
  

Can you guess what the result for q4 is?


  
`q2`                          --> add(99, 1)          --> 100
`q1`                          --> add(1, 9)           --> 10
`q3` --> multiply(`q1`, `q2`) --> multiply(10, 100)   --> 1000
`q4` --> multiply(`q3`, 5)    --> multiply(1000, 5)   --> 5000
  

Note that q1 and q2 may execute in any order, q3 may only execute when both of them finish, and q4 executes last.

This is all taken care of by qryq, so long as you define the depends for each line appropriately.

What about async?


Real World Query Queue

  
[
  {"id":"qGeocodeOrigin","depends":[],"api":"gmapsGeoLookup","qry":{"address":"36 Meadow Wood Walk, Narre Warren VIC 3805"}},
  {"id":"qGeocodeDestination","depends":[],"api":"gmapsGeoLookup","qry":{"address":"19 Bourke Street, Melbourne, VIC 3000"}},
  {"id":"qScore","depends":["qGeocodeOrigin","qGeocodeDestination"],"api":"score","qry":{
      "origin":{"address":"36 Meadow Wood Walk, Narre Warren VIC 3805","lat":"#{qGeocodeOrigin}.lat","lon":"#{qGeocodeOrigin}.lon"},
      "journeyPlanner":"melbtrans",
      "destinations":[
        {
          "fixed":true,"class":"work","weight":0.8,
          "location":{"address":"19 Bourke Street, Melbourne, VIC 3000","lat":"#{qGeocodeDestination}.lat","lon":"#{qGeocodeDestination}.lon"},
          "modes":[{"form":"transit","max":{"time":2400}}]
        }
      ]
    }
  }
]
  

From walkre


Benefits


Benefits - Dev Productivity


Benefits - Dev Productivity


Benefits - 'Net Traffic


Limitations


Limitations - REST


Limitations - Testing


Limitations - Expressions

#{previousQry}.flights.length


Implementation

Frameworks - node.js - express.js

Dependencies - Q - underscore.js


Inspiration


    
POST /api/

[
    [ 'deleteMessages', {
        idList: [ 'msg1' ]
    }],
    [ 'getMailboxMessageList', {
        mailboxName: 'Inbox',
        position: 0,
        limit: 30,
        sort: 'date descending'
    }]
]
    

Inspiration


The Play Framework at LinkedIn from Yevgeniy Brikman

The Itch


Light Bulb

I would like to process a series of data, where the output of each may be used as inputs into the others.

For example:

  
var batch = [
  {"id":"a1","depends":[],"data":{"some":"data a1"}},
  {"id":"b1","depends":["a1"],"data":{"some":"data b1"}},
  {"id":"b2","depends":["a1"],"data":{"some":"data b2"}},
  {"id":"c1","depends":["b1","b2"],"data":{"some":"data c1"}},
  {"id":"x1","depends":[],"data":{"some":"data x1"}},
];
  

This means that once a1 is complete, its output will be sent to both b1 and b2; and when these complete, both of their output will be sent to c1 (only upon both of their completion. x1 may execute in parallel with all of a1, b1, b2, and c1; and b1 may execute in parallel with b2, as no depends between them are defined.


Light Bulb

Upon completion of c1 and x1, and therefore the completion of all 5 of them, the output of all five should be returned.

We will assume that no circular dependencies are defined, and thus is a directed acyclic graph (DAG)

I would like to know how to implement this using Q, because:



Before:

{
  "origin":{"address":"36 Meadow Wood Walk, Narre Warren VIC 3805"},
  "journeyPlanner":"melbtrans",
  "destinations":[
    {
      "fixed":true,"class":"work","weight":0.8,"location":{"address":"19 Bourke Street, Melbourne, VIC 3000"},
      "modes":[{"form":"transit","max":{"time":2400}}]
    }
  ]
}

After:

[
  {"id":"qGeocodeOrigin","depends":[],"api":"gmapsGeoLookup","qry":{"address":"36 Meadow Wood Walk, Narre Warren VIC 3805"}},
  {"id":"qGeocodeDestination","depends":[],"api":"gmapsGeoLookup","qry":{"address":"19 Bourke Street, Melbourne, VIC 3000"}},
  {"id":"qScore","depends":["qGeocodeOrigin","qGeocodeDestination"],"api":"score","qry":{
      "origin":{"address":"36 Meadow Wood Walk, Narre Warren VIC 3805","lat":"#{qGeocodeOrigin}.lat","lon":"#{qGeocodeOrigin}.lon"},
      "journeyPlanner":"melbtrans",
      "destinations":[
        {
          "fixed":true,"class":"work","weight":0.8,
          "location":{"address":"19 Bourke Street, Melbourne, VIC 3000","lat":"#{qGeocodeDestination}.lat","lon":"#{qGeocodeDestination}.lon"},
          "modes":[{"form":"transit","max":{"time":2400}}]
        }
      ]
    }
  }
]


exports.score = function(deferred, qry) {
  var validateErrs = validateScore(qry);
  if (validateErrs.length > 0) {
    deferred.reject({
      msg: 'Score could not be computed',
      errors: validateErrs
    });
    return;
  }
  qry.journeyPlanner = qry.journeyPlanner || 'gmaps';
  var needGeo =
    (qry.journeyPlanner === 'melbtrans')
    || _.some(qry.destinations, function(destination) {
      destination.hasOwnProperty('fixed') && (destination.fixed === true);
    });
  var originPromises = [];
  var destinationPromises = [];
  if (needGeo) {
    //get geolocations for all locations present
    if (!qry.origin.lat || !qry.origin.lon) {
      var originGeoDeferred = Q.defer();
      exports.gmapsGeoLookup(originGeoDeferred, qry.origin);
      originPromises.push(originGeoDeferred.promise);
    }
    _.each(qry.destinations, function(destination) {
      var destGeoDeferred = Q.defer();
      exports.gmapsGeoLookup(destGeoDeferred, destination.location);
      destinationPromises.push(destGeoDeferred.promise);
    }, this);
  }
  var originPromisesDeferred = Q.defer();
  Q.allSettled(originPromises).then(function(results) {
    _.each(results, function(result) {
      if (result.state === 'fulfilled') {
        var val = result.value;
        qry.origin.lat = val.lat;
        qry.origin.lon = val.lon;
      }
      else {
        console.log('originPromises allSettled:', result.error);
      }
    }, this);
    originPromisesDeferred.resolve(qry.origin);
  });
  var destinationsPromisesDeferred = Q.defer();
  Q.allSettled(destinationPromises).then(function(results) {
    _.each(results, function(result, idx) {
      if (result.state === 'fulfilled') {
        var val = result.value;
        qry.destinations[idx].location.lat = val.lat;
        qry.destinations[idx].location.lon = val.lon;
      }
      else {
        console.log('destinationPromises allSettled:', result.error);
      }
    }, this);
    destinationsPromisesDeferred.resolve(qry.destinations);
  });
  Q.allSettled([originPromisesDeferred.promise, destinationsPromisesDeferred.promise]).then(function(results) {
    //we don't care about the results returned, because they were modified in place in the qry object
    //more importantly, we are now assured that all addresses have a lat and lon, if needGeo is true
    var scorePromises = [];

    //get the transport information from the origin to each destination using each transport mode
    var origin = qry.origin;
    _.each(qry.destinations, function(destination) {
      //TODO check that weights add up for destinations
      _.each(destination.modes, function(mode) {
        //we have origin, destination, and mode
        //TODO check that weights add up for modes
        //now work out the transport information between this origin and this destination using this mode
        var scoreDeferred = Q.defer();
        scorePromises.push(scoreDeferred.promise);
        exports.scoreOne(scoreDeferred, {
          origin: origin,
          destination: destination.location,
          mode: mode,
          journeyPlanner: qry.journeyPlanner
        });
      });
    }, this);

    Q.allSettled(scorePromises).then(function(scoreResults) {
      var orig_dest_mode = {};
      _.each(scoreResults, function(result) {
        if (result.state === 'fulfilled') {
          var score = result.value;
          orig_dest_mode[score.origin.address] = 
            orig_dest_mode[score.origin.address] || 
            {};
          orig_dest_mode[score.origin.address][score.destination.address] = 
            orig_dest_mode[score.origin.address][score.destination.address] || 
            {};
          orig_dest_mode[score.origin.address][score.destination.address][score.mode.form] = 
            orig_dest_mode[score.origin.address][score.destination.address][score.mode.form] || 
            {
              origin: score.origin,
              destination: score.destination,
              mode: score.mode,
              score: score.score
            };
        }
        else {
          console.log('scorePromises allSettled:', result.error);
        }
      });

      //some business logic
      var out = {
        /* ... */
      };
      deferred.resolve(out);
    });
  });
};

exports.score = function(deferred, qry) {
  var validateErrs = validateScore(qry);
  if (!validateErrs || validateErrs.length > 0) {
    deferred.reject({
      msg: 'Score could not be computed',
      errors: validateErrs
    });
    return;
  }
  qry.journeyPlanner = qry.journeyPlanner || 'gmaps';

  var scorePromises = [];
  //get the transport information from the origin to each destination using each transport mode
  var origin = qry.origin;
  _.each(qry.destinations, function(destination) {
    //TODO check that weights add up for destinations
    _.each(destination.modes, function(mode) {
      //we have origin, destination, and mode
      //TODO check that weights add up for modes
      //now work out the transport information between this origin and this destination using this mode
      var scoreDeferred = Q.defer();
      scorePromises.push(scoreDeferred.promise);
      exports.scoreOne(scoreDeferred, {
        origin: origin,
        destination: destination.location,
        mode: mode,
        journeyPlanner: qry.journeyPlanner
      });
    });
  });

  Q.allSettled(scorePromises).then(function(scoreResults) {
    var orig_dest_mode = {};
    _.each(scoreResults, function(result) {
      if (result.state === 'fulfilled') {
        var score = result.value;
        orig_dest_mode[score.origin.address] = 
          orig_dest_mode[score.origin.address] || 
          {};
        orig_dest_mode[score.origin.address][score.destination.address] = 
          orig_dest_mode[score.origin.address][score.destination.address] || 
          {};
        orig_dest_mode[score.origin.address][score.destination.address][score.mode.form] = 
          orig_dest_mode[score.origin.address][score.destination.address][score.mode.form] || 
          {
            origin: score.origin,
            destination: score.destination,
            mode: score.mode,
            score: score.score
          };
      }
      else {
        console.log('scorePromises allSettled:', result.error);
      }
    });

      //some business logic
      var out = {
        /* ... */
      };
      deferred.resolve(out);
  });
};

The Code


Parallel

github.com/bguiz/qryq

    
var numApiCalls = qry.length;
var apiPromises = [];
_.each(qry, function(line) {
var apiQry = line.qry;
var apiName = line.api;
var apiFunc = api[apiName];
if (!apiFunc) {
  apiFunc = api.noSuchApi;
  apiQry = apiName;
}
apiPromises.push(async(apiFunc, apiQry));
});
    

Parallel

github.com/bguiz/qryq

    
Q.allSettled(apiPromises).then(function(apiResults) {
var out = [];
_.each(apiResults, function(apiResult, idx) {
  var result = _.extend({
    id: qry[idx].id,
    api: qry[idx].api},
    apiResult);
  out.push(result);
});
deferred.resolve(out);
});
    

Sequential

github.com/bguiz/qryq

    
var numApiCalls = qry.length;
var out = [];
function sequentialLine(idx) {
var line = qry[idx];
var apiQry = line.qry;
var apiName = line.api;
var apiFunc = api[apiName];
if (!apiFunc) {
  apiFunc = api.noSuchApi;
  apiQry = apiName;
}
var promise = async(apiFunc, apiQry);
promise.then(
  /* ... */
);
}
sequentialLine(0);
    

Sequential

github.com/bguiz/qryq

promise.then(
  function(result) {
    out.push(result);
    if (idx < numApiCalls - 1) {
      sequentialLine(idx + 1);
    }
    else {
      deferred.resolve(out);
    }
  },
  function(err) {
    deferred.reject({
      error: 'Cannot process query '+apiQry.id,
      detail: err,
      incompleteResults: out
    });
  }
);

Dependent

github.com/bguiz/qryq

    
var linePromisesHash = {};
var linePromises = [];
_.each(qry, function(line) {
var apiQry = line.qry;
var apiName = line.api;
var apiFunc = api[apiName];
if (!apiFunc) {
  apiFunc = api.noSuchApi;
  apiQry = apiName;
}
var linePromise = dependentLine(line, apiFunc, linePromisesHash);
linePromises.push(linePromise);
linePromisesHash[line.id] = linePromise;
});
    

Dependent

github.com/bguiz/qryq

    
var dependentLine = function(line, apiFunc, linePromisesHash) {
  var lineDeferred = Q.defer();
  var dependsPromises = [];
  var depIds = line.depends;
  _.each(depIds, function(depId) {
    var dependPromise = linePromisesHash[depId];
    dependsPromises.push(dependPromise);
  });
  Q.allSettled(dependsPromises).then(function(dependsResults) {
    /* ... */
  });
  return lineDeferred.promise;
};
    

Dependent

github.com/bguiz/qryq

    
Q.allSettled(dependsPromises).then(function(dependsResults) {
var dependsResultsHash = {};
_.each(dependsResults, function(depResult, idx) {
  var depId = depIds[idx];
  if (depResult.state === 'fulfilled') {
    dependsResultsHash[depId] = depResult;
  }
  else {
    dependsResultsHash[depId] = null;
  }
});
var lineQryWithDepends = {};
_.extend(
  lineQryWithDepends,
  line.qry,
  {dependsResults: dependsResultsHash}
);
apiFunc(lineDeferred, lineQryWithDepends);
});
    

Dependent

github.com/bguiz/qryq

    
Q.allSettled(linePromises).then(function(lineResults) {
var out = [];
_.each(lineResults, function(lineResult, idx) {
  var lineId = qry[idx].id;
  out.push({
    id: lineId,
    response: lineResult
  });
});
deferred.resolve(out);
});
    

Horizon


    
[
    [ 'setMailboxes', {
        create: {
            '123': {
                name: 'Important'
            }
        }
    }],
    [ 'setPopLinks', {
        create: {
            '124': {
                server: 'pop.live.com',
                port: 110,
                username: 'testuser@live.com'
                password: 'letmein'
                folder: '#123'
            }
        }
    }]
]
    

Farther Horizon


Fin


Thank you

bguiz.com

@bguiz

bit.ly/qryq

github.com/bguiz/qryq

/doco/present/markdown/present.md

github.com/bguiz/walkre