"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
    return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
    return function (target, key) { decorator(target, key, paramIndex); }
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DownloadDatasetsTask = void 0;
const RopidGtfsMetadataRepository_1 = require("../../../RopidGtfsMetadataRepository");
const RopidGtfsRepository_1 = require("../../../data-access/RopidGtfsRepository");
const StaticFileRedisRepository_1 = require("../../../data-access/cache/StaticFileRedisRepository");
const DatasetEnum_1 = require("../../../helpers/DatasetEnum");
const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
const RopidMetadataModel_1 = require("../../../../shared/RopidMetadataModel");
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
const integration_engine_1 = require("@golemio/core/dist/integration-engine");
const pubsub_1 = require("@golemio/core/dist/integration-engine/data-access/pubsub");
const workers_1 = require("@golemio/core/dist/integration-engine/workers");
const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
const tsyringe_1 = require("@golemio/core/dist/shared/tsyringe");
const node_os_1 = __importDefault(require("node:os"));
const HelperTypes_1 = require("./helpers/HelperTypes");
const RopidGtfsFactory_1 = require("./helpers/RopidGtfsFactory");
const DownloadDataInputSchema_1 = require("./schema/DownloadDataInputSchema");
const constants_1 = require("../constants");
let DownloadDatasetsTask = class DownloadDatasetsTask extends workers_1.AbstractTask {
    constructor(metadataRepository, staticFileRedisRepository, ropidGtfsRepository, ropidGtfsFactory, gtfsRedisChannel, logger) {
        super(constants_1.TIMETABLE_WORKER_NAME);
        this.metadataRepository = metadataRepository;
        this.staticFileRedisRepository = staticFileRedisRepository;
        this.ropidGtfsRepository = ropidGtfsRepository;
        this.ropidGtfsFactory = ropidGtfsFactory;
        this.gtfsRedisChannel = gtfsRedisChannel;
        this.logger = logger;
        this.queueName = "downloadDatasets";
        this.queueTtl = 59 * 60 * 1000; // 59 minutes
        this.schema = DownloadDataInputSchema_1.DatasetsInputSchema;
        /**
         * Helper method - download and persist dataset
         */
        this.downloadAndPersistDatasetFiles = async (dataset) => {
            this.logger.info(`Datasets: processing ${dataset}`);
            let files = await this.ropidGtfsFactory.getDatasetMap()[dataset].datasource.getAll();
            if (!Array.isArray(files))
                files = [files];
            const dbLastModified = await this.metadataRepository.getLastModified(dataset);
            const cachedDataset = [];
            for (const file of files) {
                this.logger.info(`Datasets: processing file '${file.name}'`);
                const { data, ...meta } = file;
                if (dataset === DatasetEnum_1.DatasetEnum.PID_GTFS || dataset === DatasetEnum_1.DatasetEnum.RUN_NUMBERS) {
                    await this.staticFileRedisRepository.pipeStream(file.filepath, data);
                }
                else {
                    await this.staticFileRedisRepository.set(file.filepath, typeof data === "string" ? data : JSON.stringify(data));
                }
                const key = (0, HelperTypes_1.datasetFileModelMap)(dataset, file.name);
                const metaStates = (Array.isArray(key) ? key : [key]).map((name) => {
                    return {
                        dataset,
                        key: name,
                        type: RopidMetadataModel_1.MetaTypeEnum.STATE,
                        value: RopidMetadataModel_1.MetaStateEnum.DOWNLOADED,
                        version: dbLastModified.version + 1,
                    };
                });
                for (const metaState of metaStates) {
                    await this.metadataRepository.save(metaState);
                }
                cachedDataset.push({
                    ...meta,
                    dataset,
                });
            }
            const lastModified = await this.ropidGtfsFactory.getDatasetMap()[dataset].datasource.getLastModified();
            await this.metadataRepository.save({
                dataset,
                key: RopidMetadataModel_1.MetaDatasetInfoKeyEnum.LAST_MODIFIED,
                type: RopidMetadataModel_1.MetaTypeEnum.DATASET_INFO,
                value: lastModified,
                version: dbLastModified.version + 1,
            });
            this.logger.info(`Datasets: processing ${dataset} finished - ${JSON.stringify(cachedDataset)}`);
            return cachedDataset;
        };
    }
    async execute(data) {
        this.logger.info(`Datasets: downloading ${data.datasets}`);
        let cachedDatasets = [];
        for (const dataset of data.datasets) {
            try {
                cachedDatasets = cachedDatasets.concat(await this.downloadAndPersistDatasetFiles(dataset));
            }
            catch (err) {
                await integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "refreshPrecomputedTables", {});
                if (err instanceof golemio_errors_1.AbstractGolemioError) {
                    throw err;
                }
                throw new golemio_errors_1.GeneralError("Error while downloading GTFS datasets", this.constructor.name, err);
            }
        }
        await this.ropidGtfsRepository.createTmpTables();
        const subscriber = this.gtfsRedisChannel.createSubscriber({
            channelSuffix: node_os_1.default.hostname(),
            maxMessageCount: cachedDatasets.length,
        });
        await subscriber.subscribe();
        for (const cachedDataset of cachedDatasets) {
            integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "transformAndSaveData", cachedDataset, {
                headers: {
                    [HelperTypes_1.ORIGIN_HOSTNAME_HEADER]: node_os_1.default.hostname(),
                },
            });
        }
        // Listen for messages from workers and wait for all of them to finish
        // before continuing to the next step (replacing tables)
        try {
            await subscriber.listen(({ message, messageCount, isFinal }) => {
                this.logger.debug(`Datasets: received message: ${message} (${messageCount}/${cachedDatasets.length})`);
                if (!message.startsWith("OK")) {
                    this.logger.error(`Datasets: subscription error: ${message}`);
                    throw new golemio_errors_1.GeneralError(message, this.constructor.name);
                }
                if (isFinal) {
                    this.logger.info("Datasets: all messages received");
                }
            });
        }
        catch (err) {
            if (err instanceof golemio_errors_1.AbstractGolemioError) {
                throw err;
            }
            throw new golemio_errors_1.GeneralError("Error while listening for messages", this.constructor.name, err);
        }
        finally {
            await subscriber.unsubscribe();
            integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "checkSavedRowsAndReplaceTables", data);
        }
    }
};
exports.DownloadDatasetsTask = DownloadDatasetsTask;
exports.DownloadDatasetsTask = DownloadDatasetsTask = __decorate([
    (0, tsyringe_1.injectable)(),
    __param(0, (0, tsyringe_1.inject)(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsMetadataRepository)),
    __param(1, (0, tsyringe_1.inject)(RopidGtfsContainerToken_1.RopidGtfsContainerToken.StaticFileRedisRepository)),
    __param(2, (0, tsyringe_1.inject)(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsRepository)),
    __param(3, (0, tsyringe_1.inject)(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsFactory)),
    __param(4, (0, tsyringe_1.inject)(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RedisPubSubChannel)),
    __param(5, (0, tsyringe_1.inject)(CoreToken_1.CoreToken.Logger)),
    __metadata("design:paramtypes", [RopidGtfsMetadataRepository_1.RopidGtfsMetadataRepository,
        StaticFileRedisRepository_1.StaticFileRedisRepository,
        RopidGtfsRepository_1.RopidGtfsRepository,
        RopidGtfsFactory_1.RopidGtfsFactory,
        pubsub_1.RedisPubSubChannel, Object])
], DownloadDatasetsTask);
//# sourceMappingURL=DownloadDatasetsTask.js.map