"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const stringio_1 = require("@rauschma/stringio");
const child_process_1 = require("child_process");
const crypto_1 = require("crypto");
const BaseService_1 = __importDefault(require("../common/BaseService"));
const Datastore_1 = __importDefault(require("../common/Datastore"));
const Logger_1 = require("../common/Logger");
const Scanner_1 = __importDefault(require("./Scanner"));
const fs_extra_1 = __importDefault(require("fs-extra"));
const path_1 = __importDefault(require("path"));
const perf_hooks_1 = require("perf_hooks");
class DealPreparationWorker extends BaseService_1.default {
    constructor() {
        super(Logger_1.Category.DealPreparationWorker);
        this.PollInterval = 5000;
        this.ImmediatePollInterval = 1;
        this.workerId = (0, crypto_1.randomUUID)();
        this.startHealthCheck = this.startHealthCheck.bind(this);
        this.startPollWork = this.startPollWork.bind(this);
    }
    start() {
        if (!this.enabled) {
            this.logger.warn('Worker is not enabled. Exit now...');
        }
        this.startHealthCheck();
        this.startPollWork();
    }
    async scan(request) {
        this.logger.debug(`Started scanning.`, { id: request.id, name: request.name, path: request.path, minSize: request.minSize, maxSize: request.maxSize });
        let index = 0;
        for await (const fileList of Scanner_1.default.scan(request.path, request.minSize, request.maxSize)) {
            const existing = await Datastore_1.default.GenerationRequestModel.findOne({ datasetId: request.id, index }, { _id: 1, status: 1 });
            if (existing) {
                if (existing.status === 'created') {
                    await Datastore_1.default.GenerationRequestModel.findByIdAndDelete(existing.id, { projection: { _id: 1 } });
                }
                else {
                    continue;
                }
            }
            const generationRequest = await Datastore_1.default.GenerationRequestModel.create({
                datasetId: request.id,
                datasetName: request.name,
                path: request.path,
                outDir: request.outDir,
                tmpDir: request.tmpDir,
                index,
                status: 'created'
            });
            for (let i = 0; i < fileList.length; i += 1000) {
                await Datastore_1.default.InputFileListModel.updateOne({
                    generationId: generationRequest.id,
                    index: i / 1000
                }, {
                    $setOnInsert: {
                        fileList: fileList.slice(i, i + 1000)
                    }
                }, {
                    upsert: true,
                    projection: { _id: 1 }
                });
            }
            await Datastore_1.default.GenerationRequestModel.findByIdAndUpdate(generationRequest.id, {
                status: 'active'
            }, { projection: { _id: 1 } });
            index++;
            if ((await Datastore_1.default.ScanningRequestModel.findById(request.id))?.status === 'paused') {
                this.logger.info(`Scanning request has been paused.`);
                return;
            }
        }
        this.logger.debug(`Finished scanning. Inserted ${index} tasks.`);
        await Datastore_1.default.ScanningRequestModel.findByIdAndUpdate(request.id, { status: 'completed' });
    }
    async generate(request, input) {
        await fs_extra_1.default.mkdir(request.outDir, { recursive: true });
        this.logger.debug(`Spawning generate-car.`, { outPath: request.outDir, parentPath: request.path, tmpDir: request.tmpDir });
        let tmpDir;
        if (request.tmpDir) {
            tmpDir = path_1.default.join(request.tmpDir, (0, crypto_1.randomUUID)());
        }
        const [stdout, stderr, exitCode] = await DealPreparationWorker.invokeGenerateCar(input, request.outDir, request.path, tmpDir);
        if (tmpDir) {
            await fs_extra_1.default.rm(tmpDir, { recursive: true, force: true });
        }
        this.logger.debug(`Child process finished.`, { stdout, stderr, exitCode });
        return [stdout, stderr, exitCode];
    }
    static async invokeGenerateCar(input, outDir, p, tmpDir) {
        const cmd = await fs_extra_1.default.pathExists(path_1.default.join(__dirname, 'generate-car')) ? path_1.default.join(__dirname, 'generate-car') : 'generate-car';
        const args = ['-o', outDir, '-p', p];
        if (tmpDir) {
            args.push('-t', tmpDir);
        }
        const child = (0, child_process_1.spawn)(cmd, args, {
            stdio: ['pipe', 'pipe', 'pipe']
        });
        (async () => {
            await (0, stringio_1.streamWrite)(child.stdin, input);
            await (0, stringio_1.streamEnd)(child.stdin);
        })();
        const stdout = await (0, stringio_1.readableToString)(child.stdout);
        let stderr = '';
        child.stderr.on('data', function (chunk) {
            stderr += chunk;
        });
        try {
            await (0, stringio_1.onExit)(child);
        }
        catch (_) { }
        return [stdout, stderr, child.exitCode];
    }
    async pollScanningWork() {
        const newScanningWork = await Datastore_1.default.ScanningRequestModel.findOneAndUpdate({
            workerId: null,
            status: 'active'
        }, {
            workerId: this.workerId
        });
        if (newScanningWork) {
            this.logger.info(`${this.workerId} - Polled a new scanning request.`, { name: newScanningWork.name, id: newScanningWork.id });
            try {
                await this.scan(newScanningWork);
            }
            catch (err) {
                if (err instanceof Error) {
                    this.logger.error(`${this.workerId} - Encountered an error.`, { error: err.message });
                    await Datastore_1.default.ScanningRequestModel.findByIdAndUpdate(newScanningWork.id, { status: 'error', errorMessage: err.message });
                    return true;
                }
                throw err;
            }
        }
        return newScanningWork != null;
    }
    async pollGenerationWork() {
        const newGenerationWork = await Datastore_1.default.GenerationRequestModel.findOneAndUpdate({
            workerId: null,
            status: 'active'
        }, {
            workerId: this.workerId,
            generatedFileList: []
        }, {
            new: true
        });
        if (newGenerationWork) {
            this.logger.info(`${this.workerId} - Polled a new generation request.`, { id: newGenerationWork.id, datasetName: newGenerationWork.datasetName, index: newGenerationWork.index });
            const fileList = (await Datastore_1.default.InputFileListModel.find({
                generationId: newGenerationWork.id
            }, null, { sort: { index: 1 } })).map(r => r.fileList).flat();
            const input = JSON.stringify(fileList.map(file => ({
                Path: file.path,
                Size: file.size,
                Start: file.start,
                End: file.end
            })));
            let timeSpentInMs = perf_hooks_1.performance.now();
            const result = await this.generate(newGenerationWork, input);
            timeSpentInMs = perf_hooks_1.performance.now() - timeSpentInMs;
            // Parse the output and update the database
            const [stdout, stderr, statusCode] = result;
            if (statusCode !== 0) {
                this.logger.error(`${this.workerId} - Encountered an error.`, { stderr });
                await Datastore_1.default.GenerationRequestModel.findByIdAndUpdate(newGenerationWork.id, { status: 'error', errorMessage: stderr }, { projection: { _id: 1 } });
                return true;
            }
            const output = JSON.parse(stdout);
            const carFile = path_1.default.join(newGenerationWork.outDir, output.DataCid + '.car');
            const carFileStat = await fs_extra_1.default.stat(carFile);
            const fileMap = new Map();
            for (const fileInfo of fileList) {
                fileMap.set(path_1.default.relative(newGenerationWork.path, fileInfo.path), fileInfo);
            }
            const generatedFileList = [];
            await DealPreparationWorker.populateGeneratedFileList(fileMap, output.Ipld, [], [], generatedFileList);
            for (let i = 0; i < generatedFileList.length; i += 1000) {
                await Datastore_1.default.OutputFileListModel.updateOne({
                    generationId: newGenerationWork.id,
                    index: i / 1000
                }, {
                    $setOnInsert: {
                        generatedFileList: generatedFileList.slice(i, i + 1000)
                    }
                }, {
                    upsert: true,
                    projection: { _id: 1 }
                });
            }
            await Datastore_1.default.GenerationRequestModel.findByIdAndUpdate(newGenerationWork.id, {
                status: 'completed',
                dataCid: output.DataCid,
                pieceSize: output.PieceSize,
                pieceCid: output.PieceCid,
                carSize: carFileStat.size
            }, {
                projection: { _id: 1 }
            });
            await Datastore_1.default.InputFileListModel.deleteMany({
                generationId: newGenerationWork.id
            });
            this.logger.info(`${this.workerId} - Finished Generation of dataset.`, { id: newGenerationWork.id, datasetName: newGenerationWork.datasetName, index: newGenerationWork.index, timeSpentInMs: timeSpentInMs });
        }
        return newGenerationWork != null;
    }
    static async populateGeneratedFileList(fileMap, ipld, segments, selector, list) {
        const relPath = segments.length > 0 ? path_1.default.join(...segments) : '';
        if (ipld.Link == null) {
            const fileInfo = fileMap.get(relPath);
            list.push({
                path: relPath,
                dir: false,
                size: fileInfo.size,
                start: fileInfo.start,
                end: fileInfo.end,
                cid: ipld.Hash,
                selector: [...selector]
            });
        }
        else {
            list.push({
                path: relPath,
                dir: true,
                cid: ipld.Hash,
                selector: [...selector]
            });
            for (let linkId = 0; linkId < ipld.Link.length; ++linkId) {
                selector.push(linkId);
                segments.push(ipld.Link[linkId].Name);
                await DealPreparationWorker.populateGeneratedFileList(fileMap, ipld.Link[linkId], segments, selector, list);
                selector.pop();
                segments.pop();
            }
        }
    }
    async startPollWork() {
        let hasDoneWork = false;
        try {
            hasDoneWork = await this.pollWork();
        }
        catch (error) {
            this.logger.error(this.workerId, { error });
        }
        if (hasDoneWork) {
            setTimeout(this.startPollWork, this.ImmediatePollInterval);
        }
        else {
            setTimeout(this.startPollWork, this.PollInterval);
        }
    }
    async pollWork() {
        this.logger.debug(`${this.workerId} - Polling for work`);
        let hasDoneWork = await this.pollScanningWork();
        if (!hasDoneWork) {
            hasDoneWork = await this.pollGenerationWork();
        }
        return hasDoneWork;
    }
    async startHealthCheck() {
        await this.healthCheck();
        setTimeout(this.startHealthCheck, 5000);
    }
    async healthCheck() {
        this.logger.debug(`${this.workerId} - Sending HealthCheck`);
        await Datastore_1.default.HealthCheckModel.findOneAndUpdate({
            workerId: this.workerId
        }, {}, {
            upsert: true
        });
    }
}
exports.default = DealPreparationWorker;
//# sourceMappingURL=DealPreparationWorker.js.map