"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const body_parser_1 = __importDefault(require("body-parser"));
const config_1 = __importDefault(require("config"));
const express_1 = __importDefault(require("express"));
const fs_1 = require("fs");
const promises_1 = __importDefault(require("fs/promises"));
const path_1 = __importDefault(require("path"));
const xbytes_1 = __importDefault(require("xbytes"));
const BaseService_1 = __importDefault(require("../common/BaseService"));
const Datastore_1 = __importDefault(require("../common/Datastore"));
const Logger_1 = __importStar(require("../common/Logger"));
const ErrorCode_1 = __importDefault(require("./ErrorCode"));
const mongodb_1 = require("mongodb");
class DealPreparationService extends BaseService_1.default {
    constructor() {
        super(Logger_1.Category.DealPreparationService);
        this.app = (0, express_1.default)();
        this.handleCreatePreparationRequest = this.handleCreatePreparationRequest.bind(this);
        this.handleUpdatePreparationRequest = this.handleUpdatePreparationRequest.bind(this);
        this.handleRemovePreparationRequest = this.handleRemovePreparationRequest.bind(this);
        this.handleListPreparationRequests = this.handleListPreparationRequests.bind(this);
        this.handleGetPreparationRequest = this.handleGetPreparationRequest.bind(this);
        this.handleGetGenerationRequest = this.handleGetGenerationRequest.bind(this);
        this.handleGetGenerationManifestRequest = this.handleGetGenerationManifestRequest.bind(this);
        this.startCleanupHealthCheck = this.startCleanupHealthCheck.bind(this);
        if (!this.enabled) {
            this.logger.warn('Service is not enabled. Exit now...');
            return;
        }
        this.app.use(Logger_1.default.getExpressLogger(Logger_1.Category.DealPreparationService));
        this.app.use(body_parser_1.default.urlencoded({ extended: false }));
        this.app.use(body_parser_1.default.json());
        this.app.use(function (_req, res, next) {
            res.setHeader('Content-Type', 'application/json');
            next();
        });
        this.app.post('/preparation', this.handleCreatePreparationRequest);
        this.app.post('/preparation/:id', this.handleUpdatePreparationRequest);
        this.app.delete('/preparation/:id', this.handleRemovePreparationRequest);
        this.app.post('/preparation/:id/:generation', this.handleUpdatePreparationRequest);
        this.app.get('/preparations', this.handleListPreparationRequests);
        this.app.get('/preparation/:id', this.handleGetPreparationRequest);
        this.app.get('/generation/:dataset/:id', this.handleGetGenerationRequest);
        this.app.get('/generation/:id', this.handleGetGenerationRequest);
        this.app.get('/generation-manifest/:dataset/:id', this.handleGetGenerationManifestRequest);
        this.app.get('/generation-manifest/:id', this.handleGetGenerationManifestRequest);
    }
    start() {
        const bind = config_1.default.get('deal_preparation_service.bind');
        const port = config_1.default.get('deal_preparation_service.port');
        this.startCleanupHealthCheck();
        if (config_1.default.get('deal_preparation_service.enable_cleanup')) {
            this.cleanupIncompleteFiles();
        }
        this.app.listen(port, bind, () => {
            this.logger.info(`Service started listening at http://${bind}:${port}`);
        });
    }
    async cleanupIncompleteFiles() {
        let dirs = (await Datastore_1.default.ScanningRequestModel.find()).map(r => r.outDir);
        dirs = [...new Set(dirs)];
        for (const dir of dirs) {
            try {
                await promises_1.default.access(dir, fs_1.constants.F_OK);
                for (const file of await promises_1.default.readdir(dir)) {
                    const regex = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\.car$/;
                    if (regex.test(file)) {
                        const fullPath = path_1.default.join(dir, file);
                        this.logger.info(`Removing temporary file ${fullPath}`);
                        await promises_1.default.rm(fullPath);
                    }
                }
            }
            catch (e) {
                this.logger.warn(`${dir} cannot be read during cleanup.`);
            }
        }
        let tmpDirs = (await Datastore_1.default.ScanningRequestModel.find()).map(r => r.tmpDir);
        tmpDirs = [...new Set(tmpDirs)];
        for (const dir of tmpDirs) {
            if (dir) {
                try {
                    await promises_1.default.rm(dir, { recursive: true, force: true });
                    this.logger.info(`Removing temporary folder ${dir}`);
                }
                catch (e) {
                    this.logger.warn(`${dir} cannot be read during cleanup.`);
                }
            }
        }
    }
    async cleanupHealthCheck() {
        this.logger.debug(`Cleaning up health check table`);
        // Find all active workerId
        const workerIds = (await Datastore_1.default.HealthCheckModel.find()).map(worker => worker.workerId);
        let modified = (await Datastore_1.default.ScanningRequestModel.updateMany({ workerId: { $nin: workerIds } }, { workerId: null })).modifiedCount;
        if (modified > 0) {
            this.logger.debug(`Reset ${modified} tasks from Scanning Request table`);
        }
        modified = (await Datastore_1.default.GenerationRequestModel.updateMany({ workerId: { $nin: workerIds } }, { workerId: null })).modifiedCount;
        if (modified > 0) {
            this.logger.debug(`Reset ${modified} tasks from Generation Request table`);
        }
    }
    async startCleanupHealthCheck() {
        await this.cleanupHealthCheck();
        setTimeout(this.startCleanupHealthCheck, 5000);
    }
    async getGenerationRequest(request, response) {
        const id = request.params['id'];
        const dataset = request.params['dataset'];
        this.logger.info('Received request to get details of dataset generation.', { id, dataset });
        let found;
        const idIsInt = !isNaN(parseInt(id));
        if (mongodb_1.ObjectId.isValid(id)) {
            this.logger.debug('id is valid ObjectId');
            found = await Datastore_1.default.GenerationRequestModel.findById(id);
        }
        else if (mongodb_1.ObjectId.isValid(dataset) && idIsInt) {
            this.logger.debug('id is valid integer and dataset is valid ObjectId');
            found = await Datastore_1.default.GenerationRequestModel.findOne({ index: id, datasetId: dataset });
        }
        else if (dataset !== undefined && idIsInt) {
            this.logger.debug('id is valid integer and dataset is undefined');
            found = await Datastore_1.default.GenerationRequestModel.findOne({ index: id, datasetName: dataset });
        }
        else {
            this.sendError(response, ErrorCode_1.default.INVALID_OBJECT_ID);
            return undefined;
        }
        if (!found) {
            this.sendError(response, ErrorCode_1.default.DATASET_GENERATION_REQUEST_NOT_FOUND);
            return undefined;
        }
        return found;
    }
    static getContentsAndGroupings(generatedFileList) {
        const contents = {};
        const groupings = {};
        for (const fileInfo of generatedFileList) {
            if (fileInfo.path === '') {
                continue;
            }
            if (fileInfo.dir) {
                groupings[fileInfo.path] = fileInfo.cid;
            }
            else {
                contents[fileInfo.path] = {
                    CID: fileInfo.cid,
                    filesize: fileInfo.size
                };
                if (fileInfo.start) {
                    contents[fileInfo.path].chunkoffset = fileInfo.start;
                    contents[fileInfo.path].chunklength = fileInfo.end - fileInfo.start;
                }
            }
        }
        return [contents, groupings];
    }
    async handleGetGenerationManifestRequest(request, response) {
        const found = await this.getGenerationRequest(request, response);
        if (!found) {
            return;
        }
        if (found.status !== 'completed') {
            this.sendError(response, ErrorCode_1.default.GENERATION_NOT_COMPLETED);
            return;
        }
        const generatedFileList = (await Datastore_1.default.OutputFileListModel.find({
            generationId: found.id
        })).map(r => r.generatedFileList).flat();
        const [contents, groupings] = DealPreparationService.getContentsAndGroupings(generatedFileList);
        const result = {
            piece_cid: found.pieceCid,
            payload_cid: found.dataCid,
            raw_car_file_size: found.carSize,
            dataset: found.datasetName,
            contents,
            groupings
        };
        response.end(JSON.stringify(result));
    }
    async handleGetGenerationRequest(request, response) {
        const found = await this.getGenerationRequest(request, response);
        if (!found) {
            return;
        }
        const fileList = (await Datastore_1.default.InputFileListModel.find({
            generationId: found.id
        })).map(r => r.fileList).flat().map(r => ({
            path: r.path, size: r.size, start: r.start, end: r.end
        }));
        const generatedFileList = (await Datastore_1.default.OutputFileListModel.find({
            generationId: found.id
        })).map(r => r.generatedFileList).flat().map(r => ({
            path: r.path, size: r.size, start: r.start, end: r.end, dir: r.dir, selector: r.selector, cid: r.cid
        }));
        const result = {
            id: found.id,
            datasetId: found.datasetId,
            datasetName: found.datasetName,
            path: found.path,
            index: found.index,
            outDir: found.outDir,
            fileList,
            generatedFileList,
            workerId: found.workerId,
            status: found.status,
            errorMessage: found.errorMessage,
            dataCid: found.dataCid,
            pieceCid: found.pieceCid,
            pieceSize: found.pieceSize,
            carSize: found.carSize
        };
        response.end(JSON.stringify(result));
    }
    async handleGetPreparationRequest(request, response) {
        const id = request.params['id'];
        this.logger.info(`Received request to get details of dataset preparation request.`, { id });
        const found = mongodb_1.ObjectId.isValid(id) ? await Datastore_1.default.ScanningRequestModel.findById(id) : await Datastore_1.default.ScanningRequestModel.findOne({ name: id });
        if (!found) {
            this.sendError(response, ErrorCode_1.default.DATASET_NOT_FOUND);
            return;
        }
        const generations = await Datastore_1.default.GenerationRequestModel.find({ datasetId: found.id }, { fileList: 0, generatedFileList: 0 });
        const result = {
            id: found.id,
            name: found.name,
            path: found.path,
            minSize: found.minSize,
            maxSize: found.maxSize,
            outDir: found.outDir,
            scanningStatus: found.status,
            errorMessage: found.errorMessage,
            generationTotal: await Datastore_1.default.GenerationRequestModel.count({ datasetId: found.id }),
            generationActive: await Datastore_1.default.GenerationRequestModel.count({ datasetId: found.id, status: 'active' }),
            generationPaused: await Datastore_1.default.GenerationRequestModel.count({ datasetId: found.id, status: 'paused' }),
            generationCompleted: await Datastore_1.default.GenerationRequestModel.count({ datasetId: found.id, status: 'completed' }),
            generationError: await Datastore_1.default.GenerationRequestModel.count({ datasetId: found.id, status: 'error' }),
            generationRequests: []
        };
        for (const generation of generations) {
            result.generationRequests.push({
                id: generation.id,
                index: generation.index,
                status: generation.status,
                errorMessage: generation.errorMessage,
                dataCid: generation.dataCid,
                pieceCid: generation.pieceCid,
                pieceSize: generation.pieceSize,
                carSize: generation.carSize
            });
        }
        response.end(JSON.stringify(result));
    }
    async handleListPreparationRequests(_request, response) {
        this.logger.info('Received request to list all preparation requests.');
        const scanningRequests = await Datastore_1.default.ScanningRequestModel.find();
        const result = [];
        for (const r of scanningRequests) {
            result.push({
                id: r.id,
                name: r.name,
                path: r.path,
                minSize: r.minSize,
                maxSize: r.maxSize,
                outDir: r.outDir,
                scanningStatus: r.status,
                errorMessage: r.errorMessage,
                generationTotal: await Datastore_1.default.GenerationRequestModel.count({ datasetId: r.id }),
                generationActive: await Datastore_1.default.GenerationRequestModel.count({ datasetId: r.id, status: 'active' }),
                generationPaused: await Datastore_1.default.GenerationRequestModel.count({ datasetId: r.id, status: 'paused' }),
                generationCompleted: await Datastore_1.default.GenerationRequestModel.count({ datasetId: r.id, status: 'completed' }),
                generationError: await Datastore_1.default.GenerationRequestModel.count({ datasetId: r.id, status: 'error' })
            });
        }
        response.end(JSON.stringify(result));
    }
    async handleRemovePreparationRequest(request, response) {
        const id = request.params['id'];
        const generation = request.params['generation'];
        const { purge } = request.body;
        this.logger.info(`Received request to delete dataset preparation request.`, { id, generation, purge });
        const found = mongodb_1.ObjectId.isValid(id) ? await Datastore_1.default.ScanningRequestModel.findById(id) : await Datastore_1.default.ScanningRequestModel.findOne({ name: id });
        if (!found) {
            this.sendError(response, ErrorCode_1.default.DATASET_NOT_FOUND);
            return;
        }
        if (purge) {
            for await (const { dataCid } of Datastore_1.default.GenerationRequestModel.find({ datasetId: found.id }, { dataCid: 1 })) {
                const filename = path_1.default.join(found.outDir, dataCid + '.car');
                this.logger.info(`Removing file.`, { filename });
                await promises_1.default.rm(filename, { force: true });
            }
        }
        await found.remove();
        await Datastore_1.default.GenerationRequestModel.deleteMany({ datasetId: found.id });
        response.end();
    }
    async handleUpdatePreparationRequest(request, response) {
        const id = request.params['id'];
        const generation = request.params['generation'];
        const { action } = request.body;
        this.logger.info(`Received request to update dataset preparation request.`, { id, generation, action });
        if (!['resume', 'pause', 'retry'].includes(action)) {
            this.sendError(response, ErrorCode_1.default.ACTION_INVALID);
            return;
        }
        const found = mongodb_1.ObjectId.isValid(id) ? await Datastore_1.default.ScanningRequestModel.findById(id) : await Datastore_1.default.ScanningRequestModel.findOne({ name: id });
        if (!found) {
            this.sendError(response, ErrorCode_1.default.DATASET_NOT_FOUND);
            return;
        }
        const actionMap = {
            resume: [{ status: 'paused' }, { status: 'active' }],
            pause: [{ status: 'active' }, { status: 'paused' }],
            retry: [{ status: 'error' }, { status: 'active', errorMessage: undefined }]
        };
        const changed = (await Datastore_1.default.ScanningRequestModel.findOneAndUpdate({
            _id: found.id,
            ...actionMap[action][0]
        }, actionMap[action][1])) != null
            ? 1
            : 0;
        let changedGenerations;
        let changedGeneration;
        if (!generation) {
            changedGenerations = (await Datastore_1.default.GenerationRequestModel.updateMany({
                datasetId: found.id,
                ...actionMap[action][0]
            }, actionMap[action][1])).modifiedCount;
        }
        else {
            const generationIsInt = !isNaN(parseInt(generation));
            if (mongodb_1.ObjectId.isValid(generation)) {
                changedGeneration = (await Datastore_1.default.GenerationRequestModel.findOneAndUpdate({ _id: generation, ...actionMap[action][0] }, actionMap[action][1], { projection: { _id: 1 } }))
                    ? 1
                    : 0;
            }
            else if (generationIsInt) {
                changedGeneration = (await Datastore_1.default.GenerationRequestModel.findOneAndUpdate({ datasetId: found.id, index: generation, ...actionMap[action][0] }, actionMap[action][1], { projection: { _id: 1 } }))
                    ? 1
                    : 0;
            }
            else {
                this.sendError(response, ErrorCode_1.default.INVALID_OBJECT_ID);
                return;
            }
        }
        response.end(JSON.stringify({
            scanningRequestsChanged: changed,
            generationRequestsChanged: changedGenerations ?? changedGeneration
        }));
    }
    sendError(response, error) {
        this.logger.warn(`Error code`, { error });
        response.status(400);
        response.end(JSON.stringify({ error }));
    }
    async handleCreatePreparationRequest(request, response) {
        const { name, path, dealSize, outDir, minRatio, maxRatio, tmpDir } = request.body;
        this.logger.info(`Received request to start preparing dataset.`, { name, path, dealSize });
        const dealSizeNumber = xbytes_1.default.parseSize(dealSize);
        // Validate dealSize
        if (!DealPreparationService.AllowedDealSizes.includes(dealSizeNumber)) {
            this.sendError(response, ErrorCode_1.default.DEAL_SIZE_NOT_ALLOWED);
            return;
        }
        if (minRatio && (minRatio < 0.5 || minRatio > 0.95)) {
            this.sendError(response, ErrorCode_1.default.MIN_RATIO_INVALID);
            return;
        }
        if (maxRatio && (maxRatio < 0.5 || maxRatio > 0.95)) {
            this.sendError(response, ErrorCode_1.default.MAX_RATIO_INVALID);
            return;
        }
        if (minRatio && maxRatio && minRatio >= maxRatio) {
            this.sendError(response, ErrorCode_1.default.MAX_RATIO_INVALID);
            return;
        }
        let minSize = Math.floor(dealSizeNumber * config_1.default.get('deal_preparation_service.minDealSizeRatio'));
        if (minRatio) {
            minSize = minRatio * dealSizeNumber;
        }
        minSize = Math.round(minSize);
        let maxSize = Math.floor(dealSizeNumber * config_1.default.get('deal_preparation_service.maxDealSizeRatio'));
        if (maxRatio) {
            maxSize = maxRatio * dealSizeNumber;
        }
        maxSize = Math.round(maxSize);
        try {
            await promises_1.default.access(path, fs_1.constants.F_OK);
            await promises_1.default.access(outDir, fs_1.constants.F_OK);
            if (tmpDir) {
                await promises_1.default.access(tmpDir, fs_1.constants.F_OK);
            }
        }
        catch (_) {
            this.sendError(response, ErrorCode_1.default.PATH_NOT_ACCESSIBLE);
            return;
        }
        const scanningRequest = new Datastore_1.default.ScanningRequestModel();
        scanningRequest.name = name;
        scanningRequest.minSize = minSize;
        scanningRequest.maxSize = maxSize;
        scanningRequest.path = path;
        scanningRequest.status = 'active';
        scanningRequest.outDir = outDir;
        scanningRequest.tmpDir = tmpDir;
        try {
            await scanningRequest.save();
        }
        catch (e) {
            if (e.name === 'MongoServerError' && e.code === 11000) {
                this.sendError(response, ErrorCode_1.default.DATASET_NAME_CONFLICT);
                return;
            }
            throw e;
        }
        response.end(JSON.stringify({
            id: scanningRequest.id,
            name,
            minSize,
            maxSize,
            path,
            outDir,
            tmpDir,
            status: scanningRequest.status
        }));
    }
    static initAllowedDealSizes() {
        const result = [];
        for (let i = 8; i <= 36; i++) {
            result.push(2 ** i);
        }
        return result;
    }
}
exports.default = DealPreparationService;
DealPreparationService.AllowedDealSizes = DealPreparationService.initAllowedDealSizes();
//# sourceMappingURL=DealPreparationService.js.map