"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DataLoaderResult = exports.RecordProcessingError = exports.SystemDatabaseError = exports.InvalidPropertyTypeError = exports.MissingRequiredPropertyError = exports.EmptyFileError = exports.FileParseError = exports.UnsupportedFileExtensionError = exports.DataLoaderError = void 0;
exports.computeRecordHash = computeRecordHash;
exports.handleApplication = handleApplication;
exports.loadDataFile = loadDataFile;
const node_path_1 = require("node:path");
const node_crypto_1 = require("node:crypto");
const yaml_1 = require("yaml");
const databases_ts_1 = require("./databases.js");
const manageThreads_1 = require("../server/threads/manageThreads");
const commonErrors_js_1 = require("../utility/errors/commonErrors.js");
const hdbError_js_1 = require("../utility/errors/hdbError.js");
const harper_logger_js_1 = __importDefault(require("../utility/logging/harper_logger.js"));
const dataLoaderLogger = harper_logger_js_1.default.forComponent('dataLoader');
/** System table name for storing data loader hashes */
const DATA_LOADER_HASH_TABLE = 'hdb_dataloader_hash';
/** Lazy-initialized cache for the hash tracking table */
let _hashTrackingTable;
/**
 * Computes a deterministic hash of a record's content for tracking data file changes.
 * The hash is computed from a stable JSON representation (sorted keys) to ensure
 * the same content always produces the same hash, regardless of key order.
 *
 * @param record - The record object to hash
 * @returns A hex string hash of the record content
 */
function computeRecordHash(record) {
    // Sort keys for deterministic hashing
    const sortedKeys = Object.keys(record).sort();
    const sortedRecord = {};
    for (const key of sortedKeys) {
        sortedRecord[key] = record[key];
    }
    // Compute hash of the stable JSON representation
    const content = JSON.stringify(sortedRecord);
    return (0, node_crypto_1.createHash)('sha256').update(content).digest('hex');
}
/**
 * Gets or creates the hash tracking table in the system database.
 * Lazy-initializes the table on first access.
 */
function getHashTrackingTable(databasesRef) {
    // Always check databasesRef first (important for testing with mocks)
    if (databasesRef.system && databasesRef.system[DATA_LOADER_HASH_TABLE]) {
        return databasesRef.system[DATA_LOADER_HASH_TABLE];
    }
    // Use cached table if available
    if (_hashTrackingTable) {
        return _hashTrackingTable;
    }
    // Create the system table for tracking hashes
    _hashTrackingTable = (0, databases_ts_1.table)({
        database: 'system',
        table: DATA_LOADER_HASH_TABLE,
        attributes: [
            { name: 'id', type: 'string', isPrimaryKey: true }, // Format: "database:table:recordId"
            { name: 'hash', type: 'string' },
        ],
    });
    return _hashTrackingTable;
}
/**
 * Gets the stored hash for a record from the tracking table
 */
async function getStoredHash(database, tableName, recordId, databasesRef) {
    try {
        const trackingTable = getHashTrackingTable(databasesRef);
        const hashId = database ? `${database}:${tableName}:${recordId}` : `${tableName}:${recordId}`;
        const hashRecord = await trackingTable.get(hashId);
        return hashRecord?.hash || null;
    }
    catch (error) {
        dataLoaderLogger.error?.(`Failed to get stored hash: ${error.message}`);
        return null;
    }
}
/**
 * Stores the hash for a record in the tracking table
 */
async function storeHash(database, tableName, recordId, hash, databasesRef) {
    try {
        const trackingTable = getHashTrackingTable(databasesRef);
        const hashId = database ? `${database}:${tableName}:${recordId}` : `${tableName}:${recordId}`;
        await trackingTable.put({ id: hashId, hash });
    }
    catch (error) {
        dataLoaderLogger.error?.(`Failed to store hash: ${error.message}`);
    }
}
/**
 * Set up file handlers for data files and loads them into the appropriate tables
 */
function handleApplication(scope) {
    // Early return if this isn't worker zero
    // Currently using getWorkerIndex() over server.workerIndex to appease ts. The latter defined in manageThreads.js.
    if ((0, manageThreads_1.getWorkerIndex)() !== 0) {
        // debug and return
        dataLoaderLogger.debug?.('Skipping data loader initialization on non-primary worker');
        return;
    }
    // Handle all files that match the pattern in the config
    // Note: Using .then() instead of async/await to avoid the performance overhead
    // of additional promise wrappers created by the async/await syntax sugar
    scope.handleEntry(function handleDataLoaderEntry(entry) {
        // Return early if not adding or updating a file
        if (entry.entryType !== 'file' || entry.eventType === 'unlink') {
            return Promise.resolve();
        }
        return loadDataFile(entry, databases_ts_1.tables, databases_ts_1.databases).then((result) => {
            dataLoaderLogger.debug?.('Data loader processed file: %s: %s', (0, node_path_1.basename)(entry.absolutePath), result.message);
        });
    });
}
/**
 * This component handles data loading from YAML or JSON files into user-defined tables.
 * @param { contents, absolutePath, stats } - File entry content buffer, absolute path, and stats
 * @param tablesRef - Reference to tables object (local const for testing)
 * @param databasesRef - Reference to databases object (local const for testing)
 */
async function loadDataFile({ contents, absolutePath }, tablesRef, databasesRef) {
    const fileExt = (0, node_path_1.extname)(absolutePath) || 'unknown';
    let data;
    // Need to grab the file extension to determine how to parse the content
    try {
        if (fileExt === '.yaml' || fileExt === '.yml') {
            data = (0, yaml_1.parseDocument)(contents.toString()).toJSON();
        }
        else if (fileExt === '.json') {
            data = JSON.parse(contents.toString());
        }
        else {
            throw new UnsupportedFileExtensionError(absolutePath, fileExt);
        }
    }
    catch (error) {
        // Re-throw DataLoaderErrors
        if (error instanceof DataLoaderError) {
            throw error;
        }
        // Otherwise wrap in a FileParseError and throw
        throw new FileParseError(absolutePath, error);
    }
    // Ensure data exists. I.E. the file is not empty
    if (!data) {
        throw new EmptyFileError(absolutePath);
    }
    const { database, table: tableName, records } = data;
    // Validate the data format
    if (!tableName) {
        throw new MissingRequiredPropertyError(absolutePath, 'table');
    }
    if (!records) {
        throw new MissingRequiredPropertyError(absolutePath, 'records');
    }
    if (!Array.isArray(records)) {
        throw new InvalidPropertyTypeError(absolutePath, 'records', 'array');
    }
    // tableIdentifier is used for logging and error messages
    const tableIdentifier = database ? `${database}.${tableName}` : tableName;
    // Don't allow loading data into the system database
    if (database?.toLowerCase() === 'system') {
        throw new SystemDatabaseError(database, tableName);
    }
    try {
        // Try to get the table from global tables if it exists
        let tableRef;
        // If a database is specified, check if the table exists in that database
        if (database && databasesRef[database] && databasesRef[database][tableName]) {
            dataLoaderLogger.debug?.(`Using existing table ${tableIdentifier} from database tables`);
            tableRef = databasesRef[database][tableName];
        }
        // If no database is specified, check if the table exists in the global tables
        else if (tablesRef && tablesRef[tableName]) {
            dataLoaderLogger.debug?.(`Using existing table ${tableIdentifier} from global tables`);
            tableRef = tablesRef[tableName];
        }
        else {
            // Table doesn't exist. Try to infer the schema from the first record
            dataLoaderLogger.debug?.(`Table ${tableIdentifier} not found, creating new table`);
            // Extract attributes from the first record for the ensureTable call
            const attributes = [];
            if (records.length > 0) {
                const firstRecord = records[0];
                Object.keys(firstRecord)
                    .map((attrName) => {
                    const attr = { name: attrName, type: typeof firstRecord[attrName] };
                    // If the attribute is 'id', mark it as primary key
                    if (attrName === 'id') {
                        attr.isPrimaryKey = true;
                    }
                    return attr;
                })
                    .forEach((attr) => {
                    attributes.push(attr);
                });
            }
            tableRef = await (0, databases_ts_1.table)({
                database,
                table: tableName,
                attributes,
            });
        }
        // Process records with timestamp comparison
        // Count metrics
        const dataFIleRecords = records.length;
        let newRecords = 0;
        let updatedRecords = 0;
        let skippedRecords = 0;
        // Process each record in a batch to avoid excessive memory usage
        const batchSize = 100; // Process in batches of 100 records
        for (let i = 0; i < records.length; i += batchSize) {
            const batch = records.slice(i, i + batchSize);
            const batchPromises = [];
            for (const newRecord of batch) {
                // Wrap in an async function to handle errors individually
                batchPromises.push(async () => {
                    try {
                        // Get existing record with the same ID if it exists
                        let existingRecord = null;
                        const recordId = newRecord.id;
                        if (recordId !== undefined) {
                            existingRecord = await tableRef.get(recordId);
                        }
                        // Compute hash of the new record from the data file
                        const newRecordHash = computeRecordHash(newRecord);
                        if (!existingRecord) {
                            // If the record doesn't exist yet, insert it
                            newRecords++;
                            const result = await tableRef.put(newRecord);
                            // Store the hash in the tracking table
                            await storeHash(database, tableName, recordId, newRecordHash, databasesRef);
                            return result;
                        }
                        // Check if there's a stored hash for this record
                        const existingHash = await getStoredHash(database, tableName, recordId, databasesRef);
                        if (!existingHash) {
                            // No hash means this record wasn't loaded by the data loader
                            // (likely created via operations API or other means)
                            // Don't overwrite user-created records
                            skippedRecords++;
                            return Promise.resolve({ inserted: 0, updated: 0 });
                        }
                        // Compute hash of only the fields that exist in the data file
                        // This allows users to add extra fields without triggering a "modified" detection
                        // Note: This is a simple top-level field comparison - nested object changes
                        // within data file fields will still be detected as modifications
                        const existingRecordSubset = {};
                        for (const key of Object.keys(newRecord)) {
                            if (key in existingRecord) {
                                existingRecordSubset[key] = existingRecord[key];
                            }
                        }
                        const existingRecordHash = computeRecordHash(existingRecordSubset);
                        if (existingRecordHash !== existingHash) {
                            // The existing record's data file fields don't match the stored hash,
                            // meaning those fields were modified externally (via operations API, etc.)
                            // Don't overwrite user-modified records
                            skippedRecords++;
                            return Promise.resolve({ inserted: 0, updated: 0 });
                        }
                        // Compare hashes to detect actual content changes in the data file
                        if (newRecordHash !== existingHash) {
                            // Hash differs - the data file content has changed
                            // Use patch to update only the fields from the data file,
                            // preserving any additional fields the user may have added
                            updatedRecords++;
                            await tableRef.patch(recordId, newRecord);
                            await storeHash(database, tableName, recordId, newRecordHash, databasesRef);
                            // Return a result indicating update (patch doesn't return a value)
                            return { updated: 1 };
                        }
                        else {
                            // Hash matches - content hasn't changed, skip update
                            skippedRecords++;
                            return Promise.resolve({ inserted: 0, updated: 0 });
                        }
                    }
                    catch (error) {
                        // For individual record errors, we log but continue processing other records
                        // This allows partial success in data loading
                        if (error instanceof DataLoaderError) {
                            dataLoaderLogger.error?.(`Record processing error: ${error.message}`);
                        }
                        else {
                            const recError = new RecordProcessingError(tableIdentifier, error);
                            dataLoaderLogger.error?.(`Record processing error: ${recError.message}`);
                        }
                        // Don't throw, just return a failed operation result
                        return Promise.resolve({ inserted: 0, updated: 0, error: error.message });
                    }
                });
            }
            // Execute batch promises. Currently not doing anything about errors or the put() results.
            await Promise.all(batchPromises.map((fn) => fn()));
        }
        // Return a single result object
        if (newRecords > 0 || updatedRecords > 0) {
            let message = `Loaded ${newRecords} new and updated ${updatedRecords} records in ${tableIdentifier}`;
            if (skippedRecords > 0) {
                message += ` (${skippedRecords} records skipped)`;
            }
            dataLoaderLogger.info?.(message);
            return new DataLoaderResult(absolutePath, database, tableName, 'success', newRecords + updatedRecords, message);
        }
        else if (skippedRecords > 0) {
            const message = `All ${skippedRecords} records in ${tableIdentifier} already up-to-date`;
            dataLoaderLogger.info?.(message);
            return new DataLoaderResult(absolutePath, database, tableName, 'skipped', dataFIleRecords, message);
        }
        else {
            const message = `No records to process in ${tableIdentifier}`;
            dataLoaderLogger.info?.(message);
            return new DataLoaderResult(absolutePath, database, tableName, 'success', 0, message);
        }
    }
    catch (error) {
        // If it's already one of our custom errors, just rethrow
        if (error instanceof DataLoaderError) {
            throw error;
        }
        // Wrap and throw other errors
        throw new RecordProcessingError(tableIdentifier, error);
    }
}
/**
 * Custom errors for the dataLoader. These are thrown during startup validation to fail early
 * rather than continuing with invalid data.
 */
/**
 * Base class for DataLoader specific errors
 */
class DataLoaderError extends hdbError_js_1.ClientError {
    constructor(message, statusCode = commonErrors_js_1.HTTP_STATUS_CODES.BAD_REQUEST) {
        super(message, statusCode);
        this.name = 'DataLoaderError';
    }
}
exports.DataLoaderError = DataLoaderError;
/**
 * Error thrown when a file has an unsupported extension
 */
class UnsupportedFileExtensionError extends DataLoaderError {
    constructor(filePath, extension) {
        super(`Unsupported file extension in ${(0, node_path_1.basename)(filePath)}: ${extension}. Only YAML and JSON files are supported.`, commonErrors_js_1.HTTP_STATUS_CODES.BAD_REQUEST);
        this.name = 'UnsupportedFileExtensionError';
    }
}
exports.UnsupportedFileExtensionError = UnsupportedFileExtensionError;
/**
 * Error thrown when a file cannot be parsed
 */
class FileParseError extends DataLoaderError {
    constructor(filePath, originalError) {
        super(`Failed to parse data file ${(0, node_path_1.basename)(filePath)}: ${originalError.message}`, commonErrors_js_1.HTTP_STATUS_CODES.BAD_REQUEST);
        this.name = 'FileParseError';
    }
}
exports.FileParseError = FileParseError;
/**
 * Error thrown when a file is empty or invalid
 */
class EmptyFileError extends DataLoaderError {
    constructor(filePath) {
        super(`Data file ${(0, node_path_1.basename)(filePath)} is empty or invalid`, commonErrors_js_1.HTTP_STATUS_CODES.BAD_REQUEST);
        this.name = 'EmptyFileError';
    }
}
exports.EmptyFileError = EmptyFileError;
/**
 * Error thrown when a data file is missing required properties
 */
class MissingRequiredPropertyError extends DataLoaderError {
    constructor(filePath, property) {
        super(`Data file ${(0, node_path_1.basename)(filePath)} is missing required "${property}" property`, commonErrors_js_1.HTTP_STATUS_CODES.BAD_REQUEST);
        this.name = 'MissingRequiredPropertyError';
    }
}
exports.MissingRequiredPropertyError = MissingRequiredPropertyError;
/**
 * Error thrown when a property has an invalid type
 */
class InvalidPropertyTypeError extends DataLoaderError {
    constructor(filePath, property, expectedType) {
        super(`Data file ${(0, node_path_1.basename)(filePath)} has invalid "${property}" property, expected ${expectedType}`, commonErrors_js_1.HTTP_STATUS_CODES.BAD_REQUEST);
        this.name = 'InvalidPropertyTypeError';
    }
}
exports.InvalidPropertyTypeError = InvalidPropertyTypeError;
/**
 * Error thrown when trying to load data into the system database
 */
class SystemDatabaseError extends DataLoaderError {
    constructor(database, table) {
        super(`Cannot load data into system database: ${database}.${table}`, commonErrors_js_1.HTTP_STATUS_CODES.FORBIDDEN);
        this.name = 'SystemDatabaseError';
    }
}
exports.SystemDatabaseError = SystemDatabaseError;
/**
 * Error thrown when record processing fails
 */
class RecordProcessingError extends DataLoaderError {
    constructor(tableIdentifier, originalError) {
        super(`Failed to process record in ${tableIdentifier}: ${originalError.message}`, commonErrors_js_1.HTTP_STATUS_CODES.INTERNAL_SERVER_ERROR);
        this.name = 'RecordProcessingError';
    }
}
exports.RecordProcessingError = RecordProcessingError;
// Define the class for data loader results
class DataLoaderResult {
    #filePath; // Path to the data file
    #database; // Database name
    #table; // Table name
    #status; // Status of the operation
    #count; // Number of records processed
    #message; // Message about the operation
    constructor(filePath, database, table, status, count, message) {
        this.#filePath = filePath;
        this.#database = database || 'unknown';
        this.#table = table || 'unknown';
        this.#status = status;
        this.#count = count;
        this.#message = message;
    }
    // Getters
    get filePath() {
        return this.#filePath;
    }
    get database() {
        return this.#database;
    }
    get table() {
        return this.#table;
    }
    get status() {
        return this.#status;
    }
    get count() {
        return this.#count;
    }
    get message() {
        return this.#message;
    }
    // Methods to convert to JSON (for serialization)
    toJSON() {
        return {
            filePath: this.#filePath,
            database: this.#database,
            table: this.#table,
            status: this.#status,
            count: this.#count,
            message: this.#message,
        };
    }
}
exports.DataLoaderResult = DataLoaderResult;
//# sourceMappingURL=dataLoader.js.map