"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DownloadDatasetsTask = void 0;
const Di_1 = require("../../../../ioc/Di");
const RopidGtfsFacade_1 = require("../../../RopidGtfsFacade");
const Di_2 = require("../../../ioc/Di");
const RopidGtfsContainerToken_1 = require("../../../ioc/RopidGtfsContainerToken");
const RopidMetadataModel_1 = require("../../../../shared/RopidMetadataModel");
const CoreToken_1 = require("@golemio/core/dist/helpers/ioc/CoreToken");
const integration_engine_1 = require("@golemio/core/dist/integration-engine");
const workers_1 = require("@golemio/core/dist/integration-engine/workers");
const golemio_errors_1 = require("@golemio/core/dist/shared/golemio-errors");
const os_1 = __importDefault(require("os"));
const HelperTypes_1 = require("./helpers/HelperTypes");
const DownloadDataInputSchema_1 = require("./schema/DownloadDataInputSchema");
class DownloadDatasetsTask extends workers_1.AbstractTask {
    constructor(queuePrefix) {
        super(queuePrefix);
        this.queueName = "downloadDatasets";
        this.queueTtl = 59 * 60 * 1000; // 59 minutes
        this.schema = DownloadDataInputSchema_1.DatasetsInputSchema;
        /**
         * Helper method - download and persist dataset
         */
        this.downloadAndPersistDatasetFiles = async (dataset) => {
            this.logger.info(`Datasets: processing ${dataset}`);
            let files = await this.ropidGtfsFactory.getDatasetMap()[dataset].datasource.getAll();
            if (!Array.isArray(files))
                files = [files];
            const dbLastModified = await this.metadataRepository.getLastModified(dataset);
            const cachedDataset = [];
            for (const file of files) {
                this.logger.info(`Datasets: processing file '${file.name}'`);
                const { data, ...meta } = file;
                if (dataset === RopidGtfsFacade_1.DatasetEnum.PID_GTFS || dataset === RopidGtfsFacade_1.DatasetEnum.RUN_NUMBERS) {
                    await this.staticFileRedisRepository.pipeStream(file.filepath, data);
                }
                else {
                    await this.staticFileRedisRepository.set(file.filepath, typeof data === "string" ? data : JSON.stringify(data));
                }
                const key = (0, HelperTypes_1.datasetFileModelMap)(dataset, file.name);
                const metaStates = (Array.isArray(key) ? key : [key]).map((name) => {
                    return {
                        dataset,
                        key: name,
                        type: RopidMetadataModel_1.MetaTypeEnum.STATE,
                        value: RopidMetadataModel_1.MetaStateEnum.DOWNLOADED,
                        version: dbLastModified.version + 1,
                    };
                });
                for (const metaState of metaStates) {
                    await this.metadataRepository.save(metaState);
                }
                cachedDataset.push({
                    ...meta,
                    dataset,
                });
            }
            const lastModified = await this.ropidGtfsFactory.getDatasetMap()[dataset].datasource.getLastModified();
            await this.metadataRepository.save({
                dataset,
                key: RopidMetadataModel_1.MetaDatasetInfoKeyEnum.LAST_MODIFIED,
                type: RopidMetadataModel_1.MetaTypeEnum.DATASET_INFO,
                value: lastModified,
                version: dbLastModified.version + 1,
            });
            this.logger.info(`Datasets: processing ${dataset} finished - ${JSON.stringify(cachedDataset)}`);
            return cachedDataset;
        };
        this.logger = Di_1.PidContainer.resolve(CoreToken_1.CoreToken.Logger);
        this.gtfsRedisChannel = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RedisPubSubChannel);
        this.staticFileRedisRepository = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.StaticFileRedisRepository);
        // MetaData Model
        this.metadataRepository = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsMetadataRepository);
        this.ropidGtfsRepository = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsRepository);
    }
    async execute(data) {
        this.logger.info(`Datasets: downloading ${data.datasets}`);
        /** Ropid GTFS Dataset Map Dependencies **/
        this.ropidGtfsFactory = Di_2.RopidGtfsContainer.resolve(RopidGtfsContainerToken_1.RopidGtfsContainerToken.RopidGtfsFactory);
        let cachedDatasets = [];
        for (const dataset of data.datasets) {
            try {
                cachedDatasets = cachedDatasets.concat(await this.downloadAndPersistDatasetFiles(dataset));
            }
            catch (err) {
                await integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "refreshPrecomputedTables", {});
                if (err instanceof golemio_errors_1.AbstractGolemioError) {
                    throw err;
                }
                throw new golemio_errors_1.GeneralError("Error while downloading GTFS datasets", this.constructor.name, err);
            }
        }
        await this.ropidGtfsRepository.createTmpTables();
        const subscriber = this.gtfsRedisChannel.createSubscriber({
            channelSuffix: os_1.default.hostname(),
            maxMessageCount: cachedDatasets.length,
        });
        await subscriber.subscribe();
        for (const cachedDataset of cachedDatasets) {
            integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "transformAndSaveData", cachedDataset, {
                headers: {
                    [HelperTypes_1.ORIGIN_HOSTNAME_HEADER]: os_1.default.hostname(),
                },
            });
        }
        // Listen for messages from workers and wait for all of them to finish
        // before continuing to the next step (replacing tables)
        try {
            await subscriber.listen(({ message, messageCount, isFinal }) => {
                this.logger.debug(`Datasets: received message: ${message} (${messageCount}/${cachedDatasets.length})`);
                if (!message.startsWith("OK")) {
                    this.logger.error(`Datasets: subscription error: ${message}`);
                    throw new golemio_errors_1.GeneralError(message, this.constructor.name);
                }
                if (isFinal) {
                    this.logger.info("Datasets: all messages received");
                }
            });
        }
        catch (err) {
            if (err instanceof golemio_errors_1.AbstractGolemioError) {
                throw err;
            }
            throw new golemio_errors_1.GeneralError("Error while listening for messages", this.constructor.name, err);
        }
        finally {
            await subscriber.unsubscribe();
            integration_engine_1.QueueManager.sendMessageToExchange(this.queuePrefix, "checkSavedRowsAndReplaceTables", data);
        }
    }
}
exports.DownloadDatasetsTask = DownloadDatasetsTask;
//# sourceMappingURL=DownloadDatasetsTask.js.map