"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
Object.defineProperty(exports, "__esModule", { value: true });
const realm_sync_server_1 = require("realm-sync-server");
const Logger_1 = require("../shared/Logger");
const StatsdExporter_1 = require("../shared/StatsdExporter");
const KubernetesClient = require("kubernetes-client");
const diskusage = require("diskusage");
const events_1 = require("events");
const _1 = require(".");
const getFolderSize = require("get-folder-size");
const util_1 = require("../shared/util");
class KubernetesSyncWorker extends events_1.EventEmitter {
    constructor(config) {
        super();
        this.masterAnnotationKey = "sync.realm.io/master";
        this.slaveAnnotationKey = "sync.realm.io/slave";
        this.promoting = false;
        this.currentResourceVersion = 0;
        this.config = config;
        this.disableSlave = config.disableSlave !== undefined ? config.disableSlave : false;
        this.disableDiskStats = config.disableDiskStats !== undefined ? config.disableDiskStats : false;
    }
    start(params) {
        return __awaiter(this, void 0, void 0, function* () {
            const statsSink = params.statsSink;
            this.statsdExporter = params.statsdExporter || new StatsdExporter_1.StatsdExporter({ statsSink });
            this.logger = params.logger;
            this.namespace = this.config.kubernetesConfig.namespace;
            this.logger.debug(`Found namespace in kubernetes config: ${this.namespace}`);
            this.coreApi = new KubernetesClient.Core(this.config.kubernetesConfig).ns(this.namespace);
            this.configureDiskStats(params.statsSink);
            this.statusGauge = params.statsSink.gauge({
                name: "ros_sync_status",
                help: "Status of the sync worker",
                labelNames: ["role"],
            });
            this.statsdSocket = yield this.statsdExporter.startImporter({}, this.logger);
            const endpoints = yield this.registerCandidate();
            this.replicaCount = this.getReplicaCount(endpoints);
            yield this.reconcileEndpoints(endpoints);
            if (!params.disableWatcher) {
                const qs = { fieldSelector: `metadata.name=${this.config.endpointsName}` };
                this.watcher = new _1.ResourceWatcher({ api: this.coreApi.endpoints, qs, logger: this.logger });
                this.watcher.on("added", this.reconcileEndpoints.bind(this));
                this.watcher.on("modified", this.reconcileEndpoints.bind(this));
                this.watcher.on("deleted", (e) => {
                    this.logger.warn("Endpoints record was deleted, shutting down");
                    this.shutdown();
                });
                this.watcher.on("error", (error) => {
                    this.logger.warn("Error from watcher, restarting", { error });
                    this.watcher.start();
                });
                this.watcher.start(endpoints.metadata.resourceVersion);
            }
        });
    }
    shutdown(err) {
        return __awaiter(this, void 0, void 0, function* () {
            yield this.stopSyncWorker().catch((error) => this.logger.warn("Could not stop sync worker", { error }));
            if (this.statsdSocket) {
                yield new Promise((resolve) => this.statsdSocket.close(resolve)).catch((error) => this.logger.warn("Could not stop statsd listener socket", { error }));
                delete this.statsdSocket;
            }
            yield this.deregisterCandidate();
            if (this.statsInterval) {
                clearInterval(this.statsInterval);
                delete this.statsInterval;
            }
            this.emit("shutdown", err);
        });
    }
    reconcileEndpoints(endpoints) {
        return __awaiter(this, void 0, void 0, function* () {
            const oldReplicaCount = this.replicaCount;
            if (Number(endpoints.metadata.resourceVersion) < this.currentResourceVersion) {
                this.logger.debug(`not processing old resourceVersion: ${endpoints.metadata.resourceVersion} < ${this.currentResourceVersion}`);
                return;
            }
            this.currentResourceVersion = Number(endpoints.metadata.resourceVersion);
            this.replicaCount = this.getReplicaCount(endpoints);
            if (this.role === "master") {
                if (endpoints.metadata.annotations[this.masterAnnotationKey] !== this.config.syncWorkerId) {
                    this.logger.info("Lost master lock, becoming spare");
                    yield this.promote("spare");
                }
                if ((oldReplicaCount === 1 && this.replicaCount > 1) || (oldReplicaCount > 1 && this.replicaCount === 1)) {
                    const syncServer = yield this.syncServerPromise.catch((err) => { });
                    if (syncServer) {
                        yield syncServer.stop().catch();
                        delete this.syncServerPromise;
                    }
                    yield this.startSyncWorker();
                }
            }
            else if (this.role === "slave") {
                if (endpoints.metadata.annotations[this.masterAnnotationKey] === this.config.syncWorkerId) {
                    this.logger.info("Slave now matches master annotation, promoting to master");
                    yield this.promote("master");
                }
                else if (endpoints.metadata.annotations[this.slaveAnnotationKey] !== this.config.syncWorkerId) {
                    this.logger.info("Lost slave lock, becoming spare");
                    yield this.promote("spare");
                }
            }
            else if (!this.role || this.role === "spare") {
                if (!endpoints.metadata.annotations[this.masterAnnotationKey]) {
                    return this.assumeRole("master");
                }
                else if (endpoints.metadata.annotations[this.masterAnnotationKey] === this.config.syncWorkerId) {
                    yield this.promote("master");
                }
                else if (!endpoints.metadata.annotations[this.slaveAnnotationKey]) {
                    return this.assumeRole("slave");
                }
                else if (endpoints.metadata.annotations[this.slaveAnnotationKey] === this.config.syncWorkerId) {
                    yield this.promote("slave");
                }
                else if (this.role !== "spare") {
                    yield this.promote("spare");
                }
            }
        });
    }
    configureDiskStats(statsSink) {
        const diskUsage = statsSink.gauge({
            name: "ros_sync_disk_usage_bytes",
            help: "Realm Object Server Sync Worker disk usage in bytes",
            labelNames: [],
        });
        let diskSize, diskFree;
        if (!this.disableDiskStats) {
            diskSize = statsSink.gauge({
                name: "ros_sync_disk_size_bytes",
                help: "Realm Object Server Sync Worker disk size in bytes",
                labelNames: [],
            });
            diskFree = statsSink.gauge({
                name: "ros_sync_disk_free_bytes",
                help: "Realm Object Server Sync Worker disk free space in bytes",
                labelNames: [],
            });
        }
        this.statsInterval = setInterval(() => __awaiter(this, void 0, void 0, function* () {
            if (!this.disableDiskStats) {
                diskusage.check(this.config.dataPath, (error, result) => {
                    if (error) {
                        this.logger.warn("Error obtaining disk usage", { error });
                    }
                    else {
                        diskSize.set({}, result.total);
                        diskUsage.set({}, result.total - result.free);
                        diskFree.set({}, result.free);
                    }
                });
            }
            else {
                getFolderSize(this.config.dataPath, (error, size) => {
                    if (error) {
                        this.logger.warn("Error obtaining data directory size", { error });
                    }
                    else {
                        diskUsage.set({}, size);
                    }
                });
            }
        }), 60 * 1000);
    }
    startSyncWorker() {
        return __awaiter(this, void 0, void 0, function* () {
            if (!this.syncServerPromise) {
                const address = this.statsdSocket.address();
                const statsEndpoint = `127.0.0.1:${address.port}`;
                const params = {
                    id: this.config.syncWorkerId,
                    featureToken: this.config.featureToken,
                    dataPath: this.config.dataPath,
                    publicKeyPath: this.config.publicKeyPath,
                    listenAddress: "0.0.0.0",
                    listenPort: 0,
                    logLevel: this.config.logLevel,
                    logCallback: (levelNo, message) => this.logger.log(Logger_1.SyncLogLevel[levelNo], message),
                    enableDownloadLogCompaction: this.config.enableDownloadLogCompaction !== undefined ? this.config.enableDownloadLogCompaction : true,
                    maxDownloadSize: this.config.maxDownloadSize ? Number(this.config.maxDownloadSize) : 16 * 1024 * 1024,
                    statsEndpoint,
                    masterSlaveSharedSecret: "some-secret",
                };
                if (this.role === "master") {
                    if (this.replicaCount === 1) {
                        params.operatingMode = realm_sync_server_1.RealmSyncServerOperatingMode.MasterWithNoSlave;
                    }
                    else {
                        params.operatingMode = realm_sync_server_1.RealmSyncServerOperatingMode.MasterWithSynchronousSlave;
                        params.slaveStatusCallback = (id, upToDate) => {
                            this.logger.debug(`Slave status ${id} ${upToDate}`);
                            this.syncWorkerSlaveStatus = { id, upToDate };
                        };
                    }
                }
                else if (this.role === "slave") {
                    params.operatingMode = realm_sync_server_1.RealmSyncServerOperatingMode.Slave;
                    params.masterAddress = this.config.serviceName;
                    params.masterPort = 7800;
                }
                const syncServer = new realm_sync_server_1.RealmSyncServer(params);
                delete this.syncWorkerSlaveStatus;
                this.syncServerPromise = syncServer.start().then(() => __awaiter(this, void 0, void 0, function* () {
                    if (this.role === "master") {
                        yield this.reconcileSubsets([{
                                name: "sync",
                                port: syncServer.address().port,
                                protocol: "TCP",
                            }]);
                    }
                    return syncServer;
                }));
                yield this.syncServerPromise;
            }
            return this.syncServerPromise;
        });
    }
    stopSyncWorker() {
        return __awaiter(this, void 0, void 0, function* () {
            if (!this.syncServerPromise) {
                return;
            }
            const syncServer = yield this.syncServerPromise.catch((err) => { });
            if (!syncServer) {
                delete this.syncServerPromise;
                return;
            }
            try {
                if (this.role === "master" && this.syncWorkerSlaveStatus) {
                    while (true) {
                        if (this.syncWorkerSlaveStatus.upToDate) {
                            break;
                        }
                        this.logger.warn("Waiting for slave to synchronize before shutting down...");
                        yield new Promise((resolve) => setTimeout(resolve, 1000));
                    }
                }
                yield syncServer.stop();
                const masterAnnotationPath = `/metadata/annotations/${this.masterAnnotationKey.replace("/", "~1")}`;
                const slaveAnnotationPath = `/metadata/annotations/${this.slaveAnnotationKey.replace("/", "~1")}`;
                if (this.role === "master") {
                    if (this.syncWorkerSlaveStatus && this.syncWorkerSlaveStatus.upToDate) {
                        this.logger.info(`delegating the slave '${this.syncWorkerSlaveStatus.id}' to become master`);
                        yield this.patchEndpoints([
                            { op: "test", path: masterAnnotationPath, value: this.config.syncWorkerId },
                            { op: "replace", path: masterAnnotationPath, value: this.syncWorkerSlaveStatus.id },
                            { op: "remove", path: slaveAnnotationPath, value: this.syncWorkerSlaveStatus.id },
                            { op: "replace", path: "/subsets", value: [] },
                        ]);
                    }
                    delete this.syncWorkerSlaveStatus;
                }
                else if (this.role === "slave") {
                    this.logger.info("Relinquishing slave lock");
                    yield this.patchEndpoints([
                        { op: "test", path: slaveAnnotationPath, value: this.config.syncWorkerId },
                        { op: "remove", path: slaveAnnotationPath },
                    ]).catch((err) => {
                        this.logger.debug(`Error relinquishing slave lock: ${err.message}`);
                    });
                }
                delete this.syncServerPromise;
            }
            catch (err) {
                this.logger.warn(`Error stopping sync server: ${err.message}`);
            }
        });
    }
    assumeRole(role) {
        return __awaiter(this, void 0, void 0, function* () {
            const otherRole = role === "master" ? "slave" : "master";
            const endpoints = yield _1.retryApiRequest(() => this.coreApi.endpoints(this.config.endpointsName).get());
            const annotationKeyPrefix = "sync.realm.io";
            const roleKey = `${annotationKeyPrefix}/${role}`;
            const otherRoleKey = `${annotationKeyPrefix}/${otherRole}`;
            const annotations = endpoints.metadata.annotations || {};
            const rolePath = `/metadata/annotations/${roleKey.replace("/", "~1")}`;
            if (role === "slave" && this.disableSlave) {
                return;
            }
            const existingAnnotation = annotations[roleKey];
            if (!existingAnnotation) {
                if (annotations[otherRoleKey] === this.config.syncWorkerId) {
                    throw new Error(`Attempt to assume role while having another: ${otherRole}`);
                }
                try {
                    const ops = [
                        { op: "test", path: "/metadata/resourceVersion", value: endpoints.metadata.resourceVersion },
                        { op: "add", path: rolePath, value: this.config.syncWorkerId },
                    ];
                    const response = yield this.patchEndpoints(ops);
                    console.log("Acquired lock with response resourceVersion", response.metadata.resourceVersion);
                    return response;
                }
                catch (err) {
                    if (err.code === 409 || err.code === 500) {
                        this.logger.info(`Could not acquire endpoints ${role} lock: acquired by another worker`);
                    }
                    else {
                        this.logger.error(`Failed to patch endpoints ${role} annotation`, err);
                    }
                }
            }
            else {
                if (existingAnnotation === this.config.syncWorkerId) {
                    this.logger.debug(`Existing ${role} annotation matches us.`);
                }
                else {
                    this.logger.debug(`Existing ${role} annotation does not match us.`);
                }
            }
        });
    }
    patchEndpoints(body) {
        return __awaiter(this, void 0, void 0, function* () {
            return _1.retryApiRequest(() => this.coreApi.endpoints(this.config.endpointsName).patch({
                body, headers: { "content-type": "application/json-patch+json" },
            }));
        });
    }
    getReplicaCount(endpoints) {
        const annotations = endpoints.metadata.annotations;
        const candidates = {};
        for (const key in annotations) {
            const matches = key.match(/^candidate\.sync\.realm\.io\/(.*)$/);
            if (matches) {
                const candidateName = matches[1];
                candidates[candidateName] = JSON.parse(annotations[key]);
            }
        }
        return Object.keys(candidates).length;
    }
    reconcileSubsets(ports) {
        return __awaiter(this, void 0, void 0, function* () {
            if (this.role !== "master") {
                throw new Error(`Cannot reconcile subsets as a ${this.role}`);
            }
            const subset = {
                addresses: [{
                        ip: this.config.podIp,
                        nodeName: this.config.nodeName,
                        targetRef: this.config.podRef,
                    }],
                ports: ports || [],
            };
            yield _1.retryApiRequest(() => this.coreApi.endpoints(this.config.endpointsName).patch({
                body: { subsets: [] },
            }));
            yield util_1.delay(5000);
            return _1.retryApiRequest(() => this.coreApi.endpoints(this.config.endpointsName).patch({
                body: { subsets: [subset] },
            }));
        });
    }
    registerCandidate() {
        return __awaiter(this, void 0, void 0, function* () {
            const endpoints = yield _1.retryApiRequest(() => this.coreApi.endpoints(this.config.endpointsName).get());
            const annotations = endpoints.metadata.annotations || {};
            const groupAnnotation = `sync.realm.io/group`;
            if (!annotations[groupAnnotation]) {
                throw new Error(`Annotation '${groupAnnotation}' not found on endnpoints resource`);
            }
            if (annotations[groupAnnotation] !== this.config.syncWorkerGroup) {
                throw new Error(`Annotation '${groupAnnotation}' value '${annotations[groupAnnotation]}' does not match our group: '${this.config.syncWorkerGroup}'`);
            }
            const annotationKey = `candidate.sync.realm.io/${this.config.syncWorkerId}`;
            const annotationData = JSON.stringify({ ref: this.config.podRef });
            const path = `/metadata/annotations/${annotationKey.replace("/", "~1")}`;
            if (!annotations[annotationKey]) {
                return this.patchEndpoints([
                    { op: "add", path, value: annotationData },
                ]);
            }
            else {
                return this.patchEndpoints([
                    { op: "replace", path, value: annotationData },
                ]);
            }
        });
    }
    deregisterCandidate() {
        return __awaiter(this, void 0, void 0, function* () {
            try {
                const annotationKey = `candidate.sync.realm.io/${this.config.syncWorkerId}`;
                const path = `/metadata/annotations/${annotationKey.replace("/", "~1")}`;
                return yield this.patchEndpoints([
                    { op: "remove", path },
                ]);
            }
            catch (error) {
                this.logger.warn("Could not deregister ourselves as a candidate", { error });
            }
        });
    }
    promote(role) {
        return __awaiter(this, void 0, void 0, function* () {
            if (this.role === role) {
                return;
            }
            if (!this.promoting) {
                this.promoting = true;
                try {
                    this.logger.info(`Promoting from ${this.role} to ${role}`);
                    yield this.stopSyncWorker();
                    this.role = role;
                    if (this.role !== "spare") {
                        yield this.startSyncWorker();
                    }
                }
                catch (err) {
                    this.logger.error(`Error promoting to ${role}: ${err.message}`);
                    this.role = undefined;
                }
                finally {
                    this.promoting = false;
                    this.statusGauge.reset({});
                    if (this.role) {
                        this.statusGauge.set({ role: this.role }, 1);
                    }
                }
            }
        });
    }
}
exports.KubernetesSyncWorker = KubernetesSyncWorker;
//# sourceMappingURL=KubernetesSyncWorker.js.map