/ˈkwərik/
qry
is a NodeJs library that allows one to express a series of API queries and define the dependencies between them. These queries may be executed in parallel, in sequence, or in a directed acyclic graph.
[
{"id":"q1","depends":[],"api":"add","qry":{"a":1,"b":9}},
{"id":"q2","depends":[],"api":"add","qry":{"a":99,"b":1}},
{"id":"q3","depends":["q2","q1"],"api":"multiply","qry":{"a":"#{q1}","b":"#{q2}"}},
{"id":"q4","depends":["q3"],"api":"multiply","qry":{"a":"#{q3}","b":5}}
]
Can you guess what the result for q4
is?
`q2` --> add(99, 1) --> 100
`q1` --> add(1, 9) --> 10
`q3` --> multiply(`q1`, `q2`) --> multiply(10, 100) --> 1000
`q4` --> multiply(`q3`, 5) --> multiply(1000, 5) --> 5000
Note that q1
and q2
may execute in any order, q3
may only execute when both of them finish, and q4
executes last.
This is all taken care of by qryq, so long as you define the depends
for each line appropriately.
What about async
?
[
{"id":"qGeocodeOrigin","depends":[],"api":"gmapsGeoLookup","qry":{"address":"36 Meadow Wood Walk, Narre Warren VIC 3805"}},
{"id":"qGeocodeDestination","depends":[],"api":"gmapsGeoLookup","qry":{"address":"19 Bourke Street, Melbourne, VIC 3000"}},
{"id":"qScore","depends":["qGeocodeOrigin","qGeocodeDestination"],"api":"score","qry":{
"origin":{"address":"36 Meadow Wood Walk, Narre Warren VIC 3805","lat":"#{qGeocodeOrigin}.lat","lon":"#{qGeocodeOrigin}.lon"},
"journeyPlanner":"melbtrans",
"destinations":[
{
"fixed":true,"class":"work","weight":0.8,
"location":{"address":"19 Bourke Street, Melbourne, VIC 3000","lat":"#{qGeocodeDestination}.lat","lon":"#{qGeocodeDestination}.lon"},
"modes":[{"form":"transit","max":{"time":2400}}]
}
]
}
}
]
From walkre
qry
references the result of another qry
in the same qryq
#{previousQry}.flights.length
Frameworks - node.js - express.js
Dependencies - Q - underscore.js
POST /api/
[
[ 'deleteMessages', {
idList: [ 'msg1' ]
}],
[ 'getMailboxMessageList', {
mailboxName: 'Inbox',
position: 0,
limit: 30,
sort: 'date descending'
}]
]
I would like to process a series of data, where the output of each may be used as inputs into the others.
For example:
var batch = [
{"id":"a1","depends":[],"data":{"some":"data a1"}},
{"id":"b1","depends":["a1"],"data":{"some":"data b1"}},
{"id":"b2","depends":["a1"],"data":{"some":"data b2"}},
{"id":"c1","depends":["b1","b2"],"data":{"some":"data c1"}},
{"id":"x1","depends":[],"data":{"some":"data x1"}},
];
This means that once a1
is complete, its output will be sent to both b1
and b2
;
and when these complete, both of their output will be sent to c1
(only upon both of their completion.
x1
may execute in parallel with all of a1
, b1
, b2
, and c1
;
and b1
may execute in parallel with b2
, as no depends
between them are defined.
Upon completion of c1
and x1
, and therefore the completion of all 5 of them, the output of all five should be returned.
We will assume that no circular dependencies are defined, and thus is a directed acyclic graph (DAG)
I would like to know how to implement this using Q, because:
Q
spaghettiBefore:
{
"origin":{"address":"36 Meadow Wood Walk, Narre Warren VIC 3805"},
"journeyPlanner":"melbtrans",
"destinations":[
{
"fixed":true,"class":"work","weight":0.8,"location":{"address":"19 Bourke Street, Melbourne, VIC 3000"},
"modes":[{"form":"transit","max":{"time":2400}}]
}
]
}
After:
[
{"id":"qGeocodeOrigin","depends":[],"api":"gmapsGeoLookup","qry":{"address":"36 Meadow Wood Walk, Narre Warren VIC 3805"}},
{"id":"qGeocodeDestination","depends":[],"api":"gmapsGeoLookup","qry":{"address":"19 Bourke Street, Melbourne, VIC 3000"}},
{"id":"qScore","depends":["qGeocodeOrigin","qGeocodeDestination"],"api":"score","qry":{
"origin":{"address":"36 Meadow Wood Walk, Narre Warren VIC 3805","lat":"#{qGeocodeOrigin}.lat","lon":"#{qGeocodeOrigin}.lon"},
"journeyPlanner":"melbtrans",
"destinations":[
{
"fixed":true,"class":"work","weight":0.8,
"location":{"address":"19 Bourke Street, Melbourne, VIC 3000","lat":"#{qGeocodeDestination}.lat","lon":"#{qGeocodeDestination}.lon"},
"modes":[{"form":"transit","max":{"time":2400}}]
}
]
}
}
]
score
API before:exports.score = function(deferred, qry) {
var validateErrs = validateScore(qry);
if (validateErrs.length > 0) {
deferred.reject({
msg: 'Score could not be computed',
errors: validateErrs
});
return;
}
qry.journeyPlanner = qry.journeyPlanner || 'gmaps';
var needGeo =
(qry.journeyPlanner === 'melbtrans')
|| _.some(qry.destinations, function(destination) {
destination.hasOwnProperty('fixed') && (destination.fixed === true);
});
var originPromises = [];
var destinationPromises = [];
if (needGeo) {
//get geolocations for all locations present
if (!qry.origin.lat || !qry.origin.lon) {
var originGeoDeferred = Q.defer();
exports.gmapsGeoLookup(originGeoDeferred, qry.origin);
originPromises.push(originGeoDeferred.promise);
}
_.each(qry.destinations, function(destination) {
var destGeoDeferred = Q.defer();
exports.gmapsGeoLookup(destGeoDeferred, destination.location);
destinationPromises.push(destGeoDeferred.promise);
}, this);
}
var originPromisesDeferred = Q.defer();
Q.allSettled(originPromises).then(function(results) {
_.each(results, function(result) {
if (result.state === 'fulfilled') {
var val = result.value;
qry.origin.lat = val.lat;
qry.origin.lon = val.lon;
}
else {
console.log('originPromises allSettled:', result.error);
}
}, this);
originPromisesDeferred.resolve(qry.origin);
});
var destinationsPromisesDeferred = Q.defer();
Q.allSettled(destinationPromises).then(function(results) {
_.each(results, function(result, idx) {
if (result.state === 'fulfilled') {
var val = result.value;
qry.destinations[idx].location.lat = val.lat;
qry.destinations[idx].location.lon = val.lon;
}
else {
console.log('destinationPromises allSettled:', result.error);
}
}, this);
destinationsPromisesDeferred.resolve(qry.destinations);
});
Q.allSettled([originPromisesDeferred.promise, destinationsPromisesDeferred.promise]).then(function(results) {
//we don't care about the results returned, because they were modified in place in the qry object
//more importantly, we are now assured that all addresses have a lat and lon, if needGeo is true
var scorePromises = [];
//get the transport information from the origin to each destination using each transport mode
var origin = qry.origin;
_.each(qry.destinations, function(destination) {
//TODO check that weights add up for destinations
_.each(destination.modes, function(mode) {
//we have origin, destination, and mode
//TODO check that weights add up for modes
//now work out the transport information between this origin and this destination using this mode
var scoreDeferred = Q.defer();
scorePromises.push(scoreDeferred.promise);
exports.scoreOne(scoreDeferred, {
origin: origin,
destination: destination.location,
mode: mode,
journeyPlanner: qry.journeyPlanner
});
});
}, this);
Q.allSettled(scorePromises).then(function(scoreResults) {
var orig_dest_mode = {};
_.each(scoreResults, function(result) {
if (result.state === 'fulfilled') {
var score = result.value;
orig_dest_mode[score.origin.address] =
orig_dest_mode[score.origin.address] ||
{};
orig_dest_mode[score.origin.address][score.destination.address] =
orig_dest_mode[score.origin.address][score.destination.address] ||
{};
orig_dest_mode[score.origin.address][score.destination.address][score.mode.form] =
orig_dest_mode[score.origin.address][score.destination.address][score.mode.form] ||
{
origin: score.origin,
destination: score.destination,
mode: score.mode,
score: score.score
};
}
else {
console.log('scorePromises allSettled:', result.error);
}
});
//some business logic
var out = {
/* ... */
};
deferred.resolve(out);
});
});
};
score
API after:exports.score = function(deferred, qry) {
var validateErrs = validateScore(qry);
if (!validateErrs || validateErrs.length > 0) {
deferred.reject({
msg: 'Score could not be computed',
errors: validateErrs
});
return;
}
qry.journeyPlanner = qry.journeyPlanner || 'gmaps';
var scorePromises = [];
//get the transport information from the origin to each destination using each transport mode
var origin = qry.origin;
_.each(qry.destinations, function(destination) {
//TODO check that weights add up for destinations
_.each(destination.modes, function(mode) {
//we have origin, destination, and mode
//TODO check that weights add up for modes
//now work out the transport information between this origin and this destination using this mode
var scoreDeferred = Q.defer();
scorePromises.push(scoreDeferred.promise);
exports.scoreOne(scoreDeferred, {
origin: origin,
destination: destination.location,
mode: mode,
journeyPlanner: qry.journeyPlanner
});
});
});
Q.allSettled(scorePromises).then(function(scoreResults) {
var orig_dest_mode = {};
_.each(scoreResults, function(result) {
if (result.state === 'fulfilled') {
var score = result.value;
orig_dest_mode[score.origin.address] =
orig_dest_mode[score.origin.address] ||
{};
orig_dest_mode[score.origin.address][score.destination.address] =
orig_dest_mode[score.origin.address][score.destination.address] ||
{};
orig_dest_mode[score.origin.address][score.destination.address][score.mode.form] =
orig_dest_mode[score.origin.address][score.destination.address][score.mode.form] ||
{
origin: score.origin,
destination: score.destination,
mode: score.mode,
score: score.score
};
}
else {
console.log('scorePromises allSettled:', result.error);
}
});
//some business logic
var out = {
/* ... */
};
deferred.resolve(out);
});
};
var numApiCalls = qry.length;
var apiPromises = [];
_.each(qry, function(line) {
var apiQry = line.qry;
var apiName = line.api;
var apiFunc = api[apiName];
if (!apiFunc) {
apiFunc = api.noSuchApi;
apiQry = apiName;
}
apiPromises.push(async(apiFunc, apiQry));
});
Q.allSettled(apiPromises).then(function(apiResults) {
var out = [];
_.each(apiResults, function(apiResult, idx) {
var result = _.extend({
id: qry[idx].id,
api: qry[idx].api},
apiResult);
out.push(result);
});
deferred.resolve(out);
});
var numApiCalls = qry.length;
var out = [];
function sequentialLine(idx) {
var line = qry[idx];
var apiQry = line.qry;
var apiName = line.api;
var apiFunc = api[apiName];
if (!apiFunc) {
apiFunc = api.noSuchApi;
apiQry = apiName;
}
var promise = async(apiFunc, apiQry);
promise.then(
/* ... */
);
}
sequentialLine(0);
promise.then(
function(result) {
out.push(result);
if (idx < numApiCalls - 1) {
sequentialLine(idx + 1);
}
else {
deferred.resolve(out);
}
},
function(err) {
deferred.reject({
error: 'Cannot process query '+apiQry.id,
detail: err,
incompleteResults: out
});
}
);
var linePromisesHash = {};
var linePromises = [];
_.each(qry, function(line) {
var apiQry = line.qry;
var apiName = line.api;
var apiFunc = api[apiName];
if (!apiFunc) {
apiFunc = api.noSuchApi;
apiQry = apiName;
}
var linePromise = dependentLine(line, apiFunc, linePromisesHash);
linePromises.push(linePromise);
linePromisesHash[line.id] = linePromise;
});
var dependentLine = function(line, apiFunc, linePromisesHash) {
var lineDeferred = Q.defer();
var dependsPromises = [];
var depIds = line.depends;
_.each(depIds, function(depId) {
var dependPromise = linePromisesHash[depId];
dependsPromises.push(dependPromise);
});
Q.allSettled(dependsPromises).then(function(dependsResults) {
/* ... */
});
return lineDeferred.promise;
};
Q.allSettled(dependsPromises).then(function(dependsResults) {
var dependsResultsHash = {};
_.each(dependsResults, function(depResult, idx) {
var depId = depIds[idx];
if (depResult.state === 'fulfilled') {
dependsResultsHash[depId] = depResult;
}
else {
dependsResultsHash[depId] = null;
}
});
var lineQryWithDepends = {};
_.extend(
lineQryWithDepends,
line.qry,
{dependsResults: dependsResultsHash}
);
apiFunc(lineDeferred, lineQryWithDepends);
});
Q.allSettled(linePromises).then(function(lineResults) {
var out = [];
_.each(lineResults, function(lineResult, idx) {
var lineId = qry[idx].id;
out.push({
id: lineId,
response: lineResult
});
});
deferred.resolve(out);
});
Q
spaghetti in walkre
[
[ 'setMailboxes', {
create: {
'123': {
name: 'Important'
}
}
}],
[ 'setPopLinks', {
create: {
'124': {
server: 'pop.live.com',
port: 110,
username: 'testuser@live.com'
password: 'letmein'
folder: '#123'
}
}
}]
]
qryq