'use strict';
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.export_local = export_local;
exports.export_to_s3 = export_to_s3;
const search = __importStar(require("./search.js"));
const AWSConnector = __importStar(require("../utility/AWS/AWSConnector.js"));
const stream = __importStar(require("stream"));
const hdbUtils = __importStar(require("../utility/common_utils.js"));
const fs = __importStar(require("fs-extra"));
const path = __importStar(require("path"));
const harper_logger_ts_1 = __importDefault(require("../utility/logging/harper_logger.js"));
const util_1 = require("util");
const hdbCommon = __importStar(require("../utility/common_utils.js"));
const hdbError_ts_1 = require("../utility/errors/hdbError.js");
const commonErrors_ts_1 = require("../utility/errors/commonErrors.js");
const JSONStream_ts_1 = require("../server/serverHelpers/JSONStream.js");
let { Upload } = require('@aws-sdk/lib-storage');
const contentTypes_ts_1 = require("../server/serverHelpers/contentTypes.js");
const VALID_SEARCH_OPERATIONS = ['search_by_value', 'search_by_hash', 'sql', 'search_by_conditions'];
const VALID_EXPORT_FORMATS = ['json', 'csv'];
const JSON_TEXT = 'json';
const CSV = 'csv';
const LOCAL_JSON_EXPORT_MSG = 'Successfully exported JSON locally.';
const LOCAL_CSV_EXPORT_MSG = 'Successfully exported CSV locally.';
// Size is number of records
const S3_JSON_EXPORT_CHUNK_SIZE = 1000;
// Promisified function
const pSearchByHash = search.searchByHash;
const pSearchByValue = search.searchByValue;
const streamFinished = (0, util_1.promisify)(stream.finished);
/**
 * Allows for exporting and saving to a file system the receiving system has access to
 *
 * @param exportObject
 */
async function export_local(exportObject) {
    harper_logger_ts_1.default.trace(`export_local request to path: ${exportObject.path}, filename: ${exportObject.filename}, format: ${exportObject.format}`);
    let errorMessage = exportCoreValidation(exportObject);
    if (!hdbUtils.isEmpty(errorMessage)) {
        harper_logger_ts_1.default.error(errorMessage);
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), errorMessage, commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
    }
    if (hdbUtils.isEmpty(exportObject.path)) {
        harper_logger_ts_1.default.error(commonErrors_ts_1.HDB_ERROR_MSGS.MISSING_VALUE('path'));
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), commonErrors_ts_1.HDB_ERROR_MSGS.MISSING_VALUE('path'), commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
    }
    //we will allow for a missing filename and autogen one based on the epoch
    let filename = (hdbUtils.isEmpty(exportObject.filename) ? new Date().getTime() : exportObject.filename) +
        '.' +
        exportObject.format;
    if (exportObject.path.endsWith(path.sep)) {
        exportObject.path = exportObject.path.substring(0, exportObject.path.length - 1);
    }
    let filePath = hdbUtils.buildFolderPath(exportObject.path, filename);
    await confirmPath(exportObject.path);
    let records = await getRecords(exportObject);
    return await saveToLocal(filePath, exportObject.format, records);
}
/**
 * stats the path sent in to verify the path exists, the user has access & the path is a directory
 * @param directoryPath
 */
async function confirmPath(directoryPath) {
    harper_logger_ts_1.default.trace('in confirmPath');
    if (hdbUtils.isEmptyOrZeroLength(directoryPath)) {
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), `Invalid path: ${directoryPath}`, commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
    }
    let stats = undefined;
    try {
        stats = await fs.stat(directoryPath);
    }
    catch (err) {
        let errorMessage;
        if (err.code === 'ENOENT') {
            errorMessage = `path '${directoryPath}' does not exist`;
        }
        else if (err.code === 'EACCES') {
            errorMessage = `access to path '${directoryPath}' is denied`;
        }
        else {
            errorMessage = err.message;
        }
        harper_logger_ts_1.default.error(errorMessage);
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), errorMessage, commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
    }
    if (!stats.isDirectory()) {
        let err = `path '${directoryPath}' is not a directory, please supply a valid folder path`;
        harper_logger_ts_1.default.error(err);
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), err, commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
    }
    return true;
}
/**
 * takes the data and saves it to the file system
 * @param filePath
 * @param sourceDataFormat
 * @param data
 */
async function saveToLocal(filePath, sourceDataFormat, data) {
    harper_logger_ts_1.default.trace('in saveToLocal');
    if (hdbCommon.isEmptyOrZeroLength(filePath)) {
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), commonErrors_ts_1.HDB_ERROR_MSGS.INVALID_VALUE('file_path'), commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
    }
    if (hdbCommon.isEmptyOrZeroLength(sourceDataFormat)) {
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), commonErrors_ts_1.HDB_ERROR_MSGS.INVALID_VALUE('Source format'), commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
    }
    if (hdbCommon.isEmpty(data)) {
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), commonErrors_ts_1.HDB_ERROR_MSGS.NOT_FOUND('Data'), commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
    }
    if (sourceDataFormat === JSON_TEXT) {
        // Create a write stream to the local export file.
        let writeStream = fs.createWriteStream(filePath);
        (0, JSONStream_ts_1.streamAsJSON)(data).pipe(writeStream);
        // Wait until done. Throws if there are errors.
        await streamFinished(writeStream);
        return {
            message: LOCAL_JSON_EXPORT_MSG,
            path: filePath,
        };
    }
    else if (sourceDataFormat === CSV) {
        // Create a write stream to the local export file.
        let writeStream = fs.createWriteStream(filePath);
        const columns = data.getColumns?.();
        // Use the toCsvStream helper to convert data to CSV
        (0, contentTypes_ts_1.toCsvStream)(data, columns).pipe(writeStream);
        // Wait until done. Throws if there are errors.
        await streamFinished(writeStream);
        return {
            message: LOCAL_CSV_EXPORT_MSG,
            path: filePath,
        };
    }
    throw (0, hdbError_ts_1.handleHDBError)(new Error(), commonErrors_ts_1.HDB_ERROR_MSGS.INVALID_VALUE('format'), commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
}
/**
 *allows for exporting a result to s3
 * @param exportObject
 * @returns {*}
 */
async function export_to_s3(exportObject) {
    if (!exportObject.s3 || Object.keys(exportObject.s3).length === 0) {
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), commonErrors_ts_1.HDB_ERROR_MSGS.MISSING_VALUE('S3 object'), commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
    }
    if (hdbUtils.isEmptyOrZeroLength(exportObject.s3.aws_access_key_id)) {
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), commonErrors_ts_1.HDB_ERROR_MSGS.MISSING_VALUE('aws_access_key_id'), commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
    }
    if (hdbUtils.isEmptyOrZeroLength(exportObject.s3.aws_secret_access_key)) {
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), commonErrors_ts_1.HDB_ERROR_MSGS.MISSING_VALUE('aws_secret_access_key'), commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
    }
    if (hdbUtils.isEmptyOrZeroLength(exportObject.s3.bucket)) {
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), commonErrors_ts_1.HDB_ERROR_MSGS.MISSING_VALUE('bucket'), commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
    }
    if (hdbUtils.isEmptyOrZeroLength(exportObject.s3.key)) {
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), commonErrors_ts_1.HDB_ERROR_MSGS.MISSING_VALUE('key'), commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
    }
    if (hdbUtils.isEmptyOrZeroLength(exportObject.s3.region)) {
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), commonErrors_ts_1.HDB_ERROR_MSGS.MISSING_VALUE('region'), commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
    }
    let errorMessage = exportCoreValidation(exportObject);
    if (!hdbUtils.isEmpty(errorMessage)) {
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), errorMessage, commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
    }
    harper_logger_ts_1.default.trace(`called export_to_s3 to bucket: ${exportObject.s3.bucket} and query ${exportObject.search_operation.sql}`);
    let data;
    try {
        data = await getRecords(exportObject);
    }
    catch (err) {
        harper_logger_ts_1.default.error(err);
        throw err;
    }
    let s3 = await AWSConnector.getS3AuthObj(exportObject.s3.aws_access_key_id, exportObject.s3.aws_secret_access_key, exportObject.s3.region);
    let s3Name;
    let passThrough = new stream.PassThrough();
    if (exportObject.format === CSV) {
        s3Name = exportObject.s3.key + '.csv';
        // Create a read stream with the data using a CSV transform stream.
        const csvStream = (0, contentTypes_ts_1.toCsvStream)(data, data.getColumns?.());
        csvStream.on('error', (err) => {
            throw err;
        });
        // Pipe the data read stream through json2csv which converts it and then pipes it to a pass through which sends it to S3 upload method.
        csvStream.pipe(passThrough);
    }
    else if (exportObject.format === JSON_TEXT) {
        s3Name = exportObject.s3.key + '.json';
        // Initialize an empty read stream.
        const readableStream = new stream.Readable();
        // Pipe the read stream to a pass through, this is what sends it to the S3 upload method.
        readableStream.pipe(passThrough);
        readableStream.on('error', (err) => {
            throw err;
        });
        // Use push to add data into the read stream queue.
        readableStream.push('[');
        let dataLength = data.length;
        let chunk = '';
        // Loop through the data and build chunks to push to the read stream.
        for (const [index, record] of data.entries()) {
            let stringChunk = index === dataLength - 1 ? JSON.stringify(record) : JSON.stringify(record) + ',';
            chunk += stringChunk;
            if (index !== 0 && index % S3_JSON_EXPORT_CHUNK_SIZE === 0) {
                // Use push to add data into the read stream queue.
                readableStream.push(chunk);
                // Once the chunk has been pushed we no longer need that data. Clear it out for the next lot.
                chunk = '';
            }
        }
        // If the loop is finished and there are still items in the chunk var push it to stream.
        if (chunk.length !== 0) {
            readableStream.push(chunk);
        }
        readableStream.push(']');
        // Done writing data
        readableStream.push(null);
    }
    else {
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), commonErrors_ts_1.HDB_ERROR_MSGS.INVALID_VALUE('format'), commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
    }
    // Multipart upload to S3
    // https://github.com/aws/aws-sdk-js-v3/tree/main/lib/lib-storage
    const parallelUpload = new Upload({
        client: s3,
        params: { Bucket: exportObject.s3.bucket, Key: s3Name, Body: passThrough },
    });
    return parallelUpload.done();
}
/**
 * handles the core validation of the exportObject variable
 * @param exportObject
 * @returns {string}
 */
function exportCoreValidation(exportObject) {
    harper_logger_ts_1.default.trace('in exportCoreValidation');
    if (hdbUtils.isEmpty(exportObject.format)) {
        return 'format missing';
    }
    if (VALID_EXPORT_FORMATS.indexOf(exportObject.format) < 0) {
        return `format invalid. must be one of the following values: ${VALID_EXPORT_FORMATS.join(', ')}`;
    }
    let searchOperation = exportObject.search_operation.operation;
    if (hdbUtils.isEmpty(searchOperation)) {
        return 'search_operation.operation missing';
    }
    if (VALID_SEARCH_OPERATIONS.indexOf(searchOperation) < 0) {
        return `search_operation.operation must be one of the following values: ${VALID_SEARCH_OPERATIONS.join(', ')}`;
    }
}
let pSql;
/**
 * determines which search operation to perform and executes it.
 * @param exportObject
 */
async function getRecords(exportObject) {
    harper_logger_ts_1.default.trace('in getRecords');
    let operation;
    let errMsg = undefined;
    if (hdbCommon.isEmpty(exportObject.search_operation) ||
        hdbCommon.isEmptyOrZeroLength(exportObject.search_operation.operation)) {
        throw (0, hdbError_ts_1.handleHDBError)(new Error(), commonErrors_ts_1.HDB_ERROR_MSGS.INVALID_VALUE('Search operation'), commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
    }
    switch (exportObject.search_operation.operation) {
        case 'search_by_value':
            operation = pSearchByValue;
            break;
        case 'search_by_hash':
            operation = pSearchByHash;
            break;
        case 'search_by_conditions':
            operation = search.searchByConditions;
            break;
        case 'sql': {
            if (!pSql) {
                const sql = require('../sqlTranslator/index');
                pSql = (0, util_1.promisify)(sql.evaluateSQL);
            }
            operation = pSql;
            break;
        }
        default:
            errMsg = `Operation ${exportObject.search_operation.operation} is not support by export.`;
            harper_logger_ts_1.default.error(errMsg);
            throw (0, hdbError_ts_1.handleHDBError)(new Error(), errMsg, commonErrors_ts_1.HTTP_STATUS_CODES.BAD_REQUEST);
    }
    //in order to validate the search function and invoke permissions we need to add the hdb_user to the searchOperation
    exportObject.search_operation.hdb_user = exportObject.hdb_user;
    return operation(exportObject.search_operation);
}
//# sourceMappingURL=export.js.map