1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319 | 1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
1x
| TaskModel = require('./model/tasks').Task
Channel = require('./model/channels').Channel
Q = require("q")
logger = require("winston")
config = require("./config/config")
config.rerun = config.get('rerun')
http = require 'http'
TransactionModel = require("./model/transactions").Transaction
net = require "net"
rerunMiddleware = require "./middleware/rerunUpdateTransactionTask"
live = false
activeTasks = 0
findAndProcessAQueuedTask = ->
TaskModel.findOneAndUpdate { status: 'Queued' }, { status: 'Processing' }, { 'new': true }, (err, task) ->
if err
logger.error "An error occurred while looking for rerun tasks: #{err}"
else if task
activeTasks++
processNextTaskRound task, (err) ->
logger.error "An error occurred while processing rerun task #{task._id}: #{err}" if err
activeTasks--
# task has finished its current round, pick up the next one
if live then findAndProcessAQueuedTask()
# task has started processing, pick up the next one
if live then findAndProcessAQueuedTask()
rerunTaskProcessor = ->
if live
findAndProcessAQueuedTask()
setTimeout rerunTaskProcessor, config.rerun.processor.pollPeriodMillis
exports.start = (callback) ->
live = true
setTimeout rerunTaskProcessor, config.rerun.processor.pollPeriodMillis
logger.info "Started rerun task processor"
callback()
exports.stop = (callback) ->
live = false
waitForActiveTasks = ->
if activeTasks > 0
setTimeout waitForActiveTasks, 100
else
logger.info "Stopped rerun task processor"
callback()
waitForActiveTasks()
exports.isRunning = -> live
finalizeTaskRound = (task, callback) ->
# get latest status in case it has been changed
TaskModel.findOne { _id: task._id }, { status: 1 }, (err, result) ->
return callback err if err
# Only queue the task if still in 'Processing'
# (the status could have been changed to paused or cancelled)
if result.status is 'Processing' and task.remainingTransactions isnt 0
task.status = 'Queued'
logger.info "Round completed for rerun task ##{task._id} - #{task.remainingTransactions} transactions remaining"
else
if task.remainingTransactions is 0
task.status = 'Completed'
task.completedDate = new Date()
logger.info "Round completed for rerun task ##{task._id} - Task completed"
else
task.status = result.status
logger.info "Round completed for rerun task ##{task._id} - Task has been #{result.status}"
task.save (err) -> callback err
# Process a task.
#
# Tasks are processed in rounds:
# Each round consists of processing n transactions where n is between 1 and the task's batchSize,
# depending on how many transactions are left to process.
#
# When a round completes, the task will be marked as 'Queued' if it still has transactions remaining.
# The next available core instance will then pick up the task again for the next round.
#
# This model allows the instance the get updated information regarding the task in between rounds:
# i.e. if the server has been stopped, if the task has been paused, etc.
processNextTaskRound = (task, callback) ->
logger.debug "Processing next task round: total transactions = #{task.totalTransactions}, remainingTransactions = #{task.remainingTransactions}"
promises = []
nextI = task.transactions.length - task.remainingTransactions
for transaction in task.transactions[nextI ... nextI+task.batchSize]
do (transaction) ->
defer = Q.defer()
rerunTransaction transaction.tid, task._id, (err, response) ->
if err
transaction.tstatus = 'Failed'
transaction.error = err
logger.error "An error occurred while rerunning transaction #{transaction.tid} for task #{task._id}: #{err}"
else if response?.status is 'Failed'
transaction.tstatus = 'Failed'
transaction.error = response.message
logger.error "An error occurred while rerunning transaction #{transaction.tid} for task #{task._id}: #{err}"
else
transaction.tstatus = 'Completed'
task.remainingTransactions--
defer.resolve()
transaction.tstatus = 'Processing'
promises.push defer.promise
(Q.all promises).then ->
# Save task once transactions have been updated
task.save (err) ->
if err?
logger.error "Failed to save current task while processing round: taskID=#{task._id}, err=#{err}", err
finalizeTaskRound task, callback
rerunTransaction = (transactionID, taskID, callback) ->
rerunGetTransaction transactionID, (err, transaction) ->
return callback err if err
# setup the option object for the HTTP Request
Channel.findById transaction.channelID, (err, channel) ->
return callback err if err
logger.info "Rerunning #{channel.type} transaction"
if channel.type is 'http' or channel.type is 'polling'
rerunSetHTTPRequestOptions transaction, taskID, (err, options) ->
return callback err if err
# Run the HTTP Request with details supplied in options object
rerunHttpRequestSend options, transaction, (err, HTTPResponse) ->
return callback err, HTTPResponse
if channel.type is 'tcp' or channel.type is 'tls'
rerunTcpRequestSend channel, transaction, (err, TCPResponse) ->
return callback err if err
# Update original
ctx =
parentID : transaction._id
transactionId : transactionID
transactionStatus: TCPResponse.status
taskID : taskID
rerunMiddleware.updateOriginalTransaction ctx, (err) ->
return callback err if err
rerunMiddleware.updateTask ctx, callback
rerunGetTransaction = (transactionID, callback) ->
TransactionModel.findById transactionID, (err, transaction) ->
if not transaction?
return callback (new Error "Transaction #{transactionID} could not be found"), null
# check if 'canRerun' property is false - reject the rerun
if not transaction.canRerun
err = new Error "Transaction #{transactionID} cannot be rerun as there isn't enough information about the request"
return callback err, null
# send the transactions data in callback
return callback null, transaction
#####################################
# Construct HTTP options to be sent #
#####################################
rerunSetHTTPRequestOptions = (transaction, taskID, callback) ->
if transaction == null
err = new Error "An empty Transaction object was supplied. Aborting HTTP options configuration"
return callback err, null
logger.info('Rerun Transaction #' + transaction._id + ' - HTTP Request options being configured')
options =
hostname: config.rerun.host
port: config.rerun.httpPort
path: transaction.request.path
method: transaction.request.method
headers: transaction.request.headers
if transaction.clientID
options.headers.clientID = transaction.clientID
options.headers.parentID = transaction._id
options.headers.taskID = taskID
if transaction.request.querystring
options.path += "?"+transaction.request.querystring
return callback null, options
#####################################
# Construct HTTP options to be sent #
#####################################
#####################################
# Function for sending HTTP Request #
#####################################
rerunHttpRequestSend = (options, transaction, callback) ->
if options == null
err = new Error "An empty 'Options' object was supplied. Aborting HTTP Send Request"
return callback err, null
if transaction == null
err = new Error "An empty 'Transaction' object was supplied. Aborting HTTP Send Request"
return callback err, null
response =
body: ''
transaction: {}
logger.info('Rerun Transaction #' + transaction._id + ' - HTTP Request is being sent...')
req = http.request options, (res) ->
res.on "data", (chunk) ->
# response data
response.body += chunk
res.on "end", (err) ->
if err
response.transaction.status = "Failed"
else
response.transaction.status = "Completed"
response.status = res.statusCode
response.message = res.statusMessage
response.headers = res.headers
response.timestamp = new Date
logger.info('Rerun Transaction #' + transaction._id + ' - HTTP Response has been captured')
callback null, response
req.on "error", (err) ->
# update the status of the transaction that was processed to indicate it failed to process
response.transaction.status = "Failed" if err
response.status = 500
response.message = "Internal Server Error"
response.timestamp = new Date
callback null, response
# write data to request body
if transaction.request.method == "POST" || transaction.request.method == "PUT"
req.write transaction.request.body
req.end()
rerunTcpRequestSend = (channel, transaction, callback) ->
response =
body: ''
transaction: {}
client = new net.Socket()
client.connect channel.tcpPort, channel.tcpHost, ->
logger.info "Rerun Transaction #{transaction._id}: TCP connection established"
client.end transaction.request.body
return
client.on "data", (data) ->
response.body += data
client.on "end" , (data) ->
response.status = 200
response.transaction.status = "Completed"
response.message = ''
response.headers = {}
response.timestamp = new Date
logger.info('Rerun Transaction #' + transaction._id + ' - TCP Response has been captured')
callback null, response
return
client.on "error" , (err) ->
# update the status of the transaction that was processed to indicate it failed to process
response.transaction.status = "Failed" if err
response.status = 500
response.message = "Internal Server Error"
response.timestamp = new Date
callback err, response
#########################################################
# Export these functions when in the "test" environment #
#########################################################
if process.env.NODE_ENV == "test"
exports.rerunGetTransaction = rerunGetTransaction
exports.rerunSetHTTPRequestOptions = rerunSetHTTPRequestOptions
exports.rerunHttpRequestSend = rerunHttpRequestSend
exports.rerunTcpRequestSend = rerunTcpRequestSend
exports.findAndProcessAQueuedTask = findAndProcessAQueuedTask
|