"use strict";
var __assign = (this && this.__assign) || function () {
    __assign = Object.assign || function(t) {
        for (var s, i = 1, n = arguments.length; i < n; i++) {
            s = arguments[i];
            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
                t[p] = s[p];
        }
        return t;
    };
    return __assign.apply(this, arguments);
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __generator = (this && this.__generator) || function (thisArg, body) {
    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
    function verb(n) { return function (v) { return step([n, v]); }; }
    function step(op) {
        if (f) throw new TypeError("Generator is already executing.");
        while (g && (g = 0, op[0] && (_ = 0)), _) try {
            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
            if (y = 0, t) op = [op[0] & 2, t.value];
            switch (op[0]) {
                case 0: case 1: t = op; break;
                case 4: _.label++; return { value: op[1], done: false };
                case 5: _.label++; y = op[1]; op = [0]; continue;
                case 7: op = _.ops.pop(); _.trys.pop(); continue;
                default:
                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
                    if (t[2]) _.ops.pop();
                    _.trys.pop(); continue;
            }
            op = body.call(thisArg, _);
        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
    }
};
var __read = (this && this.__read) || function (o, n) {
    var m = typeof Symbol === "function" && o[Symbol.iterator];
    if (!m) return o;
    var i = m.call(o), r, ar = [], e;
    try {
        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
    }
    catch (error) { e = { error: error }; }
    finally {
        try {
            if (r && !r.done && (m = i["return"])) m.call(i);
        }
        finally { if (e) throw e.error; }
    }
    return ar;
};
var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
        if (ar || !(i in from)) {
            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
            ar[i] = from[i];
        }
    }
    return to.concat(ar || Array.prototype.slice.call(from));
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.WorkerServerManager = void 0;
var msgpackr_1 = require("msgpackr");
var WebSocket = require("ws");
var common_1 = require("../util/common");
var method_manager_1 = require("./method.manager");
var WorkerServerManager = /** @class */ (function () {
    function WorkerServerManager() {
        this._runningTasks = [];
        this._taskLogThresholdMs = 200;
        this._isConnected = false;
        this._lastConnectionAttemptAt = 0;
        this._lastConnectedAt = 0;
        this._lastDisconnectedAt = 0;
        this._connectionLossStartedAt = 0;
        this._lastDisconnectReason = '';
    }
    WorkerServerManager.create = function (methodManager, serverConfig) {
        var workerServerManager = new WorkerServerManager();
        workerServerManager.initialize(methodManager, serverConfig);
        return workerServerManager;
    };
    WorkerServerManager.prototype.initialize = function (methodManager, serverConfig) {
        var _this = this;
        this._methodManager = methodManager;
        this._serverConfig = serverConfig;
        if (this._methodManager && typeof this._methodManager.onReady === 'function') {
            this._methodManager.onReady(function () { return _this.startWorkerInstance(); });
        }
        else {
            this.startWorkerInstance();
        }
    };
    WorkerServerManager.prototype.markWorkerConnectionAttempt = function () {
        var now = Date.now();
        this._lastConnectionAttemptAt = now;
        if (!this._isConnected && !this._connectionLossStartedAt) {
            this._connectionLossStartedAt = now;
        }
    };
    WorkerServerManager.prototype.markWorkerConnected = function () {
        var now = Date.now();
        this._isConnected = true;
        this._lastConnectedAt = now;
        this._lastDisconnectedAt = 0;
        this._connectionLossStartedAt = 0;
        this._lastDisconnectReason = '';
    };
    WorkerServerManager.prototype.markWorkerDisconnected = function (reason) {
        var now = Date.now();
        this._isConnected = false;
        this._lastDisconnectedAt = now;
        if (!this._connectionLossStartedAt) {
            this._connectionLossStartedAt = now;
        }
        this._lastDisconnectReason = String(reason || 'Worker websocket disconnected.');
    };
    WorkerServerManager.prototype.getConnectionHealth = function () {
        return {
            isConnected: this._isConnected,
            lastConnectionAttemptAt: this._lastConnectionAttemptAt,
            lastConnectedAt: this._lastConnectedAt,
            lastDisconnectedAt: this._lastDisconnectedAt,
            connectionLossStartedAt: this._connectionLossStartedAt,
            currentOutageMs: (!this._isConnected && this._connectionLossStartedAt)
                ? Math.max(0, Date.now() - this._connectionLossStartedAt)
                : 0,
            lastDisconnectReason: this._lastDisconnectReason
        };
    };
    WorkerServerManager.prototype.startWorkerInstance = function () {
        var _this = this;
        var workerIndex = encodeURIComponent(String(process.env.WORKER_INDEX || ''));
        var workerInstance = encodeURIComponent(String(process.env.NODE_APP_INSTANCE || ''));
        var workerToken = encodeURIComponent(String(this._serverConfig['WORKER_TOKEN'] || ''));
        var wsUrl = this._serverConfig['SERVER_URL'] + '/websocket?workerToken=' + workerToken + '&workerIndex=' + workerIndex + '&workerInstance=' + workerInstance;
        var ws = new WebSocket(wsUrl);
        this.markWorkerConnectionAttempt();
        var lastComm = new Date();
        var interval = null;
        var missedPongs = 0;
        var heartbeatIntervalMs = 15000;
        var maxMissedPongs = 2;
        var maxSilenceMs = heartbeatIntervalMs * (maxMissedPongs + 1);
        var openTimeout = null;
        var opened = false;
        // Set timeout if the socket never opens
        openTimeout = setTimeout(function () {
            if (!opened) {
                _this.markWorkerDisconnected('WebSocket connection timeout.');
                console.error(new Date(), 'WebSocket connection timeout. Retrying...');
                ws.terminate(); // force close if still connecting
            }
        }, 10000); // 10 seconds timeout
        ws.on('open', function () {
            opened = true;
            clearTimeout(openTimeout);
            _this.markWorkerConnected();
            console.log(new Date(), 'Connected to main server as worker', process.env.WORKER_INDEX, process.env.NODE_APP_INSTANCE);
            _this.sendWorkerResponse(ws, 'ping');
            interval = setInterval(function () {
                var now = Date.now();
                var last = lastComm ? lastComm.getTime() : 0;
                var silenceMs = last ? now - last : maxSilenceMs + 1;
                if (silenceMs > maxSilenceMs || missedPongs > maxMissedPongs) {
                    _this.markWorkerDisconnected('Worker websocket heartbeat timed out.');
                    ws.close();
                    return;
                }
                missedPongs += 1;
                _this.sendWorkerResponse(ws, 'ping');
            }, heartbeatIntervalMs);
        });
        ws.on('message', function (rawData) { return __awaiter(_this, void 0, void 0, function () {
            var msg_1, messageBuffer, chunks, view, heartbeat, msg;
            return __generator(this, function (_a) {
                switch (_a.label) {
                    case 0:
                        lastComm = new Date();
                        if (!(typeof rawData === 'string')) return [3 /*break*/, 5];
                        if (this._methodManager.getEnableDebug()) {
                            console.log(new Date(), 'Message Recv', rawData);
                        }
                        if (!(rawData === 'ping')) return [3 /*break*/, 1];
                        if (this._methodManager.getEnableDebug()) {
                            console.log(new Date(), 'Recv Ping, Sending Pong');
                        }
                        this.sendWorkerResponse(ws, 'pong');
                        return [3 /*break*/, 4];
                    case 1:
                        if (!(rawData === 'pong')) return [3 /*break*/, 2];
                        if (this._methodManager.getEnableDebug()) {
                            console.log(new Date(), 'Recv Pong');
                        }
                        missedPongs = 0;
                        return [3 /*break*/, 4];
                    case 2:
                        try {
                            msg_1 = JSON.parse(rawData, common_1.dateReviver);
                        }
                        catch (e) {
                            console.error('Worker parse error', e);
                            return [2 /*return*/];
                        }
                        if (!(msg_1.type === 'task')) return [3 /*break*/, 4];
                        return [4 /*yield*/, this.handleIncomingTask(ws, msg_1)];
                    case 3:
                        _a.sent();
                        _a.label = 4;
                    case 4: return [2 /*return*/];
                    case 5:
                        if (Buffer.isBuffer(rawData)) {
                            messageBuffer = rawData;
                        }
                        else if (Array.isArray(rawData)) {
                            chunks = rawData;
                            messageBuffer = Buffer.concat(chunks);
                        }
                        else if (rawData instanceof ArrayBuffer) {
                            messageBuffer = Buffer.from(rawData);
                        }
                        else if (ArrayBuffer.isView(rawData)) {
                            view = rawData;
                            messageBuffer = Buffer.from(view.buffer, view.byteOffset, view.byteLength);
                        }
                        else {
                            messageBuffer = Buffer.from(rawData);
                        }
                        if (this._methodManager.getEnableDebug()) {
                            console.log(new Date(), 'Message Recv (binary)', messageBuffer.length);
                        }
                        if (messageBuffer.length === 4) {
                            heartbeat = messageBuffer.toString('utf8');
                            if (heartbeat === 'ping') {
                                if (this._methodManager.getEnableDebug()) {
                                    console.log(new Date(), 'Recv Ping (binary), Sending Pong');
                                }
                                this.sendWorkerResponse(ws, 'pong');
                                return [2 /*return*/];
                            }
                            else if (heartbeat === 'pong') {
                                if (this._methodManager.getEnableDebug()) {
                                    console.log(new Date(), 'Recv Pong (binary)');
                                }
                                missedPongs = 0;
                                return [2 /*return*/];
                            }
                        }
                        try {
                            msg = (0, msgpackr_1.unpack)(messageBuffer);
                        }
                        catch (e) {
                            console.error('Worker binary parse error', e);
                            return [2 /*return*/];
                        }
                        if (!(msg.type === 'task')) return [3 /*break*/, 7];
                        return [4 /*yield*/, this.handleIncomingTask(ws, msg)];
                    case 6:
                        _a.sent();
                        _a.label = 7;
                    case 7: return [2 /*return*/];
                }
            });
        }); });
        ws.on('close', function (code, reason) {
            var closeReason = reason && reason.length
                ? "Worker websocket closed (".concat(code, "): ").concat(reason.toString())
                : "Worker websocket closed (".concat(code, ")");
            _this.markWorkerDisconnected(closeReason);
            setTimeout(function () {
                _this.startWorkerInstance();
            }, 1000);
            if (interval) {
                clearInterval(interval);
            }
            clearTimeout(openTimeout);
        });
        ws.on('error', function (err) {
            var message = (err === null || err === void 0 ? void 0 : err.message) || '';
            var statusMatch = message.match(/Unexpected server response:\s*(\d{3})/);
            var statusCode = statusMatch ? statusMatch[1] : null;
            var isRestartNoise = statusCode === '502';
            if (statusCode && !isRestartNoise) {
                console.error(new Date(), "Worker WS error: Failure ".concat(statusCode));
            }
            else if (!isRestartNoise) {
                console.error(new Date(), 'Worker WS error:', err);
            }
            else if (_this._methodManager.getEnableDebug()) {
                console.log(new Date(), 'Worker WS reconnecting after main restart', message);
            }
            _this.markWorkerDisconnected(message || 'Worker websocket error.');
            ws.close();
        });
    };
    WorkerServerManager.prototype.handleIncomingTask = function (ws, data) {
        return __awaiter(this, void 0, void 0, function () {
            var taskId, messageId, method, params, userContext, receivedAt, queuedAt, dispatchedAt, queueWaitMs, dispatchDelayMs, missingMethod, error_1, timedOut, timeoutHandle, managerThis, result, packedResult, meta, publicationPayload, publicationMeta, isPublicationResponse, payload, err_1, errorAt_1, runMs_1, totalMs_1, errorAt, runMs, totalMs;
            var _a;
            var _this = this;
            var _b;
            return __generator(this, function (_c) {
                switch (_c.label) {
                    case 0:
                        taskId = data.taskId, messageId = data.messageId, method = data.method, params = data.params, userContext = data.userContext;
                        receivedAt = Date.now();
                        queuedAt = data.queuedAt;
                        dispatchedAt = data.dispatchedAt;
                        queueWaitMs = queuedAt && dispatchedAt ? (0, common_1.round)(dispatchedAt - queuedAt) : (queuedAt ? (0, common_1.round)(receivedAt - queuedAt) : null);
                        dispatchDelayMs = dispatchedAt ? (0, common_1.round)(receivedAt - dispatchedAt) : null;
                        this._runningTasks.push(taskId);
                        missingMethod = !method || !this._methodManager.getMethod(method);
                        if (!(!taskId || !method || missingMethod)) return [3 /*break*/, 5];
                        if (!missingMethod) return [3 /*break*/, 4];
                        console.error(new Date(), 'No method in method manager for handleIncomingTask worker server', method);
                        if (!this._methodManager.getMethod('insertErrorLog')) return [3 /*break*/, 4];
                        _c.label = 1;
                    case 1:
                        _c.trys.push([1, 3, , 4]);
                        return [4 /*yield*/, this._methodManager.callMethod('insertErrorLog', "No Method in worker server handleIncomingTask: ".concat(method, " - ").concat(JSON.stringify(data, null, 2)))];
                    case 2:
                        _c.sent();
                        return [3 /*break*/, 4];
                    case 3:
                        error_1 = _c.sent();
                        console.error(new Date(), 'insertErrorLog failed in worker-server', error_1);
                        return [3 /*break*/, 4];
                    case 4:
                        // console.log('Invalid task message received', data);
                        this.logWorkerEvent('workerTaskInvalid', {
                            taskId: taskId,
                            method: method,
                            messageId: messageId,
                            queueWaitMs: queueWaitMs,
                            dispatchDelayMs: dispatchDelayMs,
                            userContext: userContext,
                            params: params
                        });
                        this.sendWorkerResponse(ws, {
                            type: 'taskComplete',
                            taskId: taskId,
                            messageId: messageId,
                            error: true,
                            result: 'Invalid task'
                        });
                        return [2 /*return*/];
                    case 5:
                        timedOut = false;
                        timeoutHandle = setTimeout(function () { return __awaiter(_this, void 0, void 0, function () {
                            var timeoutAt, runMs, totalMs;
                            return __generator(this, function (_a) {
                                switch (_a.label) {
                                    case 0:
                                        timedOut = true;
                                        timeoutAt = Date.now();
                                        runMs = (0, common_1.round)(timeoutAt - receivedAt);
                                        totalMs = queuedAt ? (0, common_1.round)(timeoutAt - queuedAt) : null;
                                        console.error(new Date(), 'Worker timed out on task:', taskId, 'Method:', method);
                                        this.logWorkerEvent('workerTaskTimeout', {
                                            taskId: taskId,
                                            method: method,
                                            messageId: messageId,
                                            runMs: runMs,
                                            queueWaitMs: queueWaitMs,
                                            dispatchDelayMs: dispatchDelayMs,
                                            totalMs: totalMs,
                                            userContext: userContext,
                                            params: params
                                        });
                                        this.sendWorkerResponse(ws, {
                                            type: 'taskComplete',
                                            taskId: taskId,
                                            messageId: messageId,
                                            error: true,
                                            result: 'Task timed out'
                                        });
                                        return [4 /*yield*/, this._methodManager.callMethod('insertErrorLog', "Timeout in Method: ".concat(method, " - ").concat(JSON.stringify(data, null, 2)))];
                                    case 1:
                                        _a.sent();
                                        return [2 /*return*/];
                                }
                            });
                        }); }, ((_b = this._methodManager.getMethod(method)) === null || _b === void 0 ? void 0 : _b.timeoutOverride) || (1000 * 60 * 2));
                        _c.label = 6;
                    case 6:
                        _c.trys.push([6, 8, , 10]);
                        managerThis = Object.assign({}, this._methodManager, method_manager_1.MethodManager.prototype, {
                            id_user: (userContext === null || userContext === void 0 ? void 0 : userContext.id_user) || '',
                            user: (userContext === null || userContext === void 0 ? void 0 : userContext.user) || '',
                            id_ws: (userContext === null || userContext === void 0 ? void 0 : userContext.id_ws) || ''
                        });
                        if (this._methodManager.getEnableDebug()) {
                            console.log(new Date(), 'Running method', method);
                        }
                        return [4 /*yield*/, (_a = this._methodManager.callMethod).call.apply(_a, __spreadArray([managerThis, method], __read(params), false))];
                    case 7:
                        result = _c.sent();
                        packedResult = null;
                        meta = null;
                        publicationPayload = undefined;
                        publicationMeta = null;
                        isPublicationResponse = !!(result && result.__publicationResponse);
                        try {
                            if (isPublicationResponse) {
                                publicationPayload = result.payload;
                                publicationMeta = {
                                    type: 'publication',
                                    publication: result.publication,
                                    subscriptionData: result.subscriptionData,
                                    userSpecific: result.userSpecific,
                                    snapshot: result.snapshot
                                };
                            }
                            if (isPublicationResponse) {
                                meta = publicationMeta;
                                packedResult = (0, msgpackr_1.pack)(publicationPayload);
                                result = null;
                            }
                            else {
                                packedResult = (0, msgpackr_1.pack)(result);
                            }
                        }
                        catch (packErr) {
                            console.error(new Date(), 'Worker pack error', packErr);
                            if (isPublicationResponse) {
                                meta = publicationMeta;
                                result = publicationPayload;
                            }
                        }
                        if (this._methodManager.getEnableDebug()) {
                            console.log(new Date(), 'Finished method', method);
                        }
                        if (!timedOut) {
                            clearTimeout(timeoutHandle);
                            payload = {
                                type: 'taskComplete',
                                taskId: taskId,
                                messageId: messageId,
                                error: false,
                                result: packedResult ? null : result,
                                packedResult: packedResult,
                                encoding: 'msgpack',
                                meta: meta
                            };
                            this.sendWorkerResponse(ws, payload);
                        }
                        this._runningTasks = this._runningTasks.filter(function (a) { return a !== taskId; });
                        if (this._methodManager.getEnableDebug()) {
                            console.log(new Date(), 'Done with Task', JSON.stringify(this._runningTasks, null, 2));
                        }
                        return [3 /*break*/, 10];
                    case 8:
                        err_1 = _c.sent();
                        if (!timedOut) {
                            clearTimeout(timeoutHandle);
                            errorAt_1 = Date.now();
                            runMs_1 = (0, common_1.round)(errorAt_1 - receivedAt);
                            totalMs_1 = queuedAt ? (0, common_1.round)(errorAt_1 - queuedAt) : null;
                            console.error('Worker failed task:', taskId, 'Method:', method, err_1);
                            err_1.message = 'Worker failed task:' + taskId + ' Method:' + method + ' - ' + err_1.message;
                            this.logWorkerEvent('workerTaskError', {
                                taskId: taskId,
                                method: method,
                                messageId: messageId,
                                runMs: runMs_1,
                                queueWaitMs: queueWaitMs,
                                dispatchDelayMs: dispatchDelayMs,
                                totalMs: totalMs_1,
                                userContext: userContext,
                                params: params,
                                error: err_1
                            });
                            this.sendWorkerResponse(ws, {
                                type: 'taskComplete',
                                taskId: taskId,
                                messageId: messageId,
                                error: true,
                                result: err_1 || 'Unknown error'
                            });
                            throw err_1;
                        }
                        errorAt = Date.now();
                        runMs = (0, common_1.round)(errorAt - receivedAt);
                        totalMs = queuedAt ? (0, common_1.round)(errorAt - queuedAt) : null;
                        this.logWorkerEvent('workerTaskError', {
                            taskId: taskId,
                            method: method,
                            messageId: messageId,
                            runMs: runMs,
                            queueWaitMs: queueWaitMs,
                            dispatchDelayMs: dispatchDelayMs,
                            totalMs: totalMs,
                            userContext: userContext,
                            params: params,
                            error: err_1
                        });
                        console.error(new Date(), "Error in Method: ".concat(method, " - ").concat(JSON.stringify(data, null, 2)));
                        return [4 /*yield*/, this._methodManager.callMethod('insertErrorLog', "Error in Method: ".concat(method, " - ").concat(JSON.stringify(data, null, 2)))];
                    case 9:
                        _c.sent();
                        this._runningTasks = this._runningTasks.filter(function (a) { return a !== taskId; });
                        return [3 /*break*/, 10];
                    case 10: return [2 /*return*/];
                }
            });
        });
    };
    WorkerServerManager.prototype.logWorkerEvent = function (event, details) {
        console.log(this.safeStringify(__assign({ ts: new Date().toISOString(), event: event }, details)));
    };
    WorkerServerManager.prototype.safeStringify = function (value) {
        try {
            var seen_1 = new WeakSet();
            return JSON.stringify(value, function (key, val) {
                if (key === '__proto__') {
                    return undefined;
                }
                if (typeof val === 'bigint') {
                    return val.toString();
                }
                if (val instanceof Error) {
                    return {
                        name: val.name,
                        message: val.message,
                        stack: val.stack
                    };
                }
                if (val && typeof val === 'object') {
                    if (seen_1.has(val)) {
                        return '[Circular]';
                    }
                    seen_1.add(val);
                }
                return val;
            });
        }
        catch (err) {
            return JSON.stringify({
                error: 'safeStringifyFailed',
                message: (err === null || err === void 0 ? void 0 : err.message) || String(err)
            });
        }
    };
    WorkerServerManager.prototype.sendWorkerResponse = function (ws, payload) {
        if (this._methodManager.getEnableDebug()) {
            console.log(new Date(), 'Sending', typeof payload === 'string' ? payload : '[binary]');
        }
        if (ws && ws.readyState === WebSocket.OPEN) {
            try {
                if (typeof payload === 'string') {
                    ws.send(payload);
                }
                else {
                    ws.send((0, msgpackr_1.pack)(payload));
                }
            }
            catch (err) {
                console.error('Failed to send worker response:', err);
            }
        }
        else if (ws) {
            ws.close();
        }
    };
    return WorkerServerManager;
}());
exports.WorkerServerManager = WorkerServerManager;

//# sourceMappingURL=worker-server.manager.js.map
