How to run a historic query and deliver data to a push destination.
Require the DataSift library - choose one of these:
var DataSift = require('datasift-node'); // When running from NPM package
var DataSift = require('../lib/datasift'); // When running within datasift-node repository
Create a DataSift client object - insert your API credentials:
var ds = new DataSift('YOUR_USERNAME', 'YOUR_APIKEY');
Calculate start and end time (as UNIX timestamp) for our query (start 48 hours ago, duration 2 hours)
var startTime = parseInt((new Date).getTime()/1000) - (7200 * 48);
var endTime = parseInt(startTime + 3600);
This method gets a list of all historic queries for the user:
function getAll() {
ds.historics.get(function(err, response) {
if (err)
console.log(err);
else
{
console.log("Historic queries: " + JSON.stringify(response));
checkStatus();
}
});
}
This method checks the historic data coverage for the query period, then compiles CSDL for the query:
function checkStatus() {
console.log("Checking data coverage for our query period");
ds.historics.status({
"start": startTime,
"end": endTime
}, function(err,response) {
if (err)
console.log(err);
else
{
console.log("Historic status for query period: " + JSON.stringify(response));
compileStream();
}
});
}
This method compiles CSDL into a hash for the query, then prepares the query:
function compileStream() {
ds.compile({ 'csdl': 'interaction.content contains "datasift"' }, function (err, response) {
if (err)
console.log(err);
else
{
console.log("Compiled filter hash: " + response.hash);
prepareQuery(response.hash);
}
});
}
Prepares the historic query, then creates the subscription for data delivery:
function prepareQuery(hash) {
ds.historics.prepare({
"hash": hash,
"start": startTime,
"end": endTime,
"sources": "twitter",
"name": "Example historic query"
}, function(err, response) {
if (err)
console.log(err);
else
{
console.log("Prepared query: " + response.id);
createSubscription(response.id);
}
});
}
Creates a Pull subscription for data delivery, then starts the query:
function createSubscription(historicsId) {
ds.push.create({
'historics_id': historicsId,
'name': "Example historics subscription",
'output_type': 'pull'
}, function(err, response) {
if (err)
console.log(err);
else
{
subscriptionId = response.id;
console.log("Created subscription ID: " + subscriptionId);
start(historicsId);
}
});
}
Starts the query running, then updates the query name:
function start(historicsId) {
ds.historics.start({
"id": historicsId
}, function(err, response) {
if (err)
console.log(err);
else
{
console.log("Started historic query.");
update(historicsId);
}
});
}
Updates the query’s name, then gets the query’s details:
function update(historicsId) {
ds.historics.update({
"id": historicsId,
"name": "Updated historic query"
}, function(err, response) {
if (err)
console.log(err);
else
{
console.log("Updated historic query.");
get(historicsId);
}
});
}
Gets the historic query’s details, then pauses the query - In a real world solution you would allow the query to run and deliver all the data!
function get(historicsId) {
ds.historics.get({
"id": historicsId
}, function(err, response) {
if (err)
console.log(err);
else
{
console.log("Historic query details: " + JSON.stringify(response));
pause(historicsId);
}
});
}
Pauses the query, then resumes it:
function pause(historicsId) {
ds.historics.pause({
"id": historicsId,
"reason": "Pausing example query"
}, function(err, response) {
if (err)
console.log(err);
else
{
console.log("Paused historic query.");
resume(historicsId);
}
});
}
Resumes the query, then stops it:
function resume(historicsId) {
ds.historics.resume({
"id": historicsId
}, function(err, response) {
if (err)
console.log(err);
else
{
console.log("Resumed historic query.");
stop(historicsId);
}
});
}
Stops the query running, then deletes it:
function stop(historicsId) {
ds.historics.stop({
"id": historicsId
}, function(err, response) {
if (err)
console.log(err);
else
{
console.log("Stopped historic query.");
deleteHistoric(historicsId);
}
});
}
Deletes the historic query:
function deleteHistoric(historicsId) {
ds.historics.delete({
"id": historicsId
}, function(err, response) {
if (err)
console.log(err);
else
{
console.log("Deleted historic query.");
}
});
}
Finally we start the process, comprising of:
getAll();