"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.PulseCronAWS = void 0;
const client_s3_1 = require("@aws-sdk/client-s3");
const date_fns_1 = require("date-fns");
const semaphore_1 = require("../../../util/semaphore");
const pulseCronCommon_1 = require("./pulseCronCommon");
class PulseCronAWS {
    constructor({ bucketName, client, prefix, maxConcurrentDownloads, maxKeys, startTime, endTime, earlyTermination, region, samplingByCount, samplingByPercentage, samplingCount, samplingPercentage, validExtensions = Object.values(pulseCronCommon_1.PULSE_CRON_EXTENSION_TYPES).flat(), applicationId }) {
        this.currentBatchFiles = null;
        this.semaphore = new semaphore_1.Semaphore(10);
        this.bucketName = bucketName;
        this.client = client;
        this.prefix = prefix;
        this.maxConcurrentDownloads = maxConcurrentDownloads;
        this.maxKeys = maxKeys;
        this.startTime = typeof startTime === 'string' ? (0, date_fns_1.parseISO)(startTime.replace(' ', 'T') + 'Z') : startTime;
        this.endTime = typeof endTime === 'string' ? (0, date_fns_1.parseISO)(endTime.replace(' ', 'T') + 'Z') : endTime;
        this.earlyTermination = earlyTermination;
        this.region = region;
        this.samplingByCount = samplingByCount;
        this.samplingByPercentage = samplingByPercentage;
        this.samplingCount = samplingCount;
        this.samplingPercentage = samplingPercentage;
        this.validExtensions = validExtensions;
        this.applicationId = applicationId;
        this.semaphore.acquire(async () => {
            console.log("Acquired semaphore");
        });
    }
    // Check if file is a transcript file for an audio file
    isTranscriptFile(key) {
        const lowerKey = key.toLowerCase();
        const transcriptExtensions = ['.json', '.log', '.txt'];
        // Check if it ends with transcript extension
        const hasTranscriptExt = transcriptExtensions.some(ext => lowerKey.endsWith(ext));
        if (!hasTranscriptExt) {
            return false;
        }
        // Check if there's a corresponding audio file with same base name
        const baseName = (0, pulseCronCommon_1.getBaseFileName)(key);
        const audioExtensions = ['.wav', '.mp3', '.ogg', '.flac', '.pcm16'];
        return audioExtensions.some(audioExt => {
            const potentialAudioFile = baseName + audioExt;
            return this.fileExists(potentialAudioFile);
        });
    }
    // Check if a file exists in the current batch (this will be populated during processing)
    fileExists(fileName) {
        // This will be implemented to check against the current batch of files
        return this.currentBatchFiles?.has(fileName.toLowerCase()) || false;
    }
    // Get transcript content for an audio file
    async getTranscriptContent(audioFileName) {
        const baseName = (0, pulseCronCommon_1.getBaseFileName)(audioFileName);
        const transcriptExtensions = ['.json', '.log', '.txt'];
        for (const ext of transcriptExtensions) {
            const transcriptKey = baseName + ext;
            try {
                const command = new client_s3_1.GetObjectCommand({
                    Bucket: this.bucketName,
                    Key: transcriptKey
                });
                const response = await this.client.send(command);
                const content = await response.Body?.transformToString();
                return content || null;
            }
            catch {
                // Transcript file doesn't exist, try next extension
                continue;
            }
        }
        return null;
    }
    // Perform randomized sampling with guaranteed uniqueness
    getRandomizedSample(logs, targetCount) {
        if (targetCount >= logs.length) {
            return logs; // Return all logs if target count is greater than or equal to total
        }
        const selectedLogs = [];
        const selectedIndices = new Set(); // Track selected indices for O(1) lookup
        for (let i = 0; i < targetCount; i++) {
            let attempts = 0;
            const maxAttempts = logs.length * 2; // Prevent infinite loops
            while (attempts < maxAttempts) {
                const index = Math.floor(Math.random() * logs.length);
                if (!selectedIndices.has(index)) {
                    selectedLogs.push(logs[index]);
                    selectedIndices.add(index);
                    break;
                }
                attempts++;
            }
            // If we couldn't find a unique index, break to avoid infinite loop
            if (attempts >= maxAttempts) {
                console.warn(`Could not find unique log after ${maxAttempts} attempts. Selected ${selectedLogs.length} logs instead of ${targetCount}`);
                break;
            }
        }
        return selectedLogs;
    }
    // Get filename from full path (extract just the filename, not the path)
    getFileNameFromPath(fullPath) {
        const lastSlashIndex = fullPath.lastIndexOf('/');
        return lastSlashIndex === -1 ? fullPath : fullPath.substring(lastSlashIndex + 1);
    }
    // Get logs with early termination and smart pagination
    async getLogs() {
        const targetCount = this.samplingByCount ? this.samplingCount : null;
        console.log(`Filtering logs: startTime=${this.startTime.toISOString()}, endTime=${this.endTime.toISOString()}`);
        console.log(`Time range: ${this.endTime.getTime() - this.startTime.getTime()}ms (${(this.endTime.getTime() - this.startTime.getTime()) / (1000 * 60 * 60)} hours)`);
        let allValidLogs = [];
        const seenFileNames = new Set(); // Track seen filenames for deduplication (GLOBAL across all batches)
        const seenKeys = new Set(); // Track seen full keys for additional protection
        let continuationToken;
        let totalScanned = 0;
        let foundEnoughForCount = false;
        try {
            // Phase 1: Fast metadata collection with early termination
            do {
                const startTime = Date.now();
                const command = new client_s3_1.ListObjectsV2Command({
                    Bucket: this.bucketName,
                    Prefix: this.prefix,
                    MaxKeys: this.maxKeys,
                    ContinuationToken: continuationToken
                });
                const response = await this.client.send(command);
                totalScanned += response.Contents?.length || 0;
                if (response.Contents) {
                    // Filter and sort in one pass with file extension validation
                    const totalFiles = response.Contents.length;
                    const timeFilteredFiles = response.Contents.filter(obj => {
                        const lastModified = new Date(obj.LastModified);
                        return lastModified >= this.startTime && lastModified <= this.endTime;
                    });
                    // First, populate currentBatchFiles for transcript detection
                    this.currentBatchFiles = new Set(timeFilteredFiles.map(obj => (obj.Key || '').toLowerCase()));
                    const extensionFilteredFiles = timeFilteredFiles.filter(obj => {
                        const key = obj.Key || '';
                        // Check for full key duplicates first (most specific)
                        if (seenKeys.has(key)) {
                            return false; // Skip duplicate full key
                        }
                        // Exclude transcript files from valid logs
                        if (this.isTranscriptFile(key)) {
                            return false;
                        }
                        if (!(0, pulseCronCommon_1.hasValidExtension)(key, this.validExtensions)) {
                            return false;
                        }
                        // Check for duplicates based on filename (GLOBAL across all batches)
                        const fileName = this.getFileNameFromPath(key);
                        if (seenFileNames.has(fileName)) {
                            return false; // Skip duplicate filename
                        }
                        return true;
                    });
                    const duplicatesRemoved = timeFilteredFiles.length - extensionFilteredFiles.length - (timeFilteredFiles.length - timeFilteredFiles.filter(obj => {
                        const key = obj.Key || '';
                        return !this.isTranscriptFile(key) && (0, pulseCronCommon_1.hasValidExtension)(key, this.validExtensions);
                    }).length);
                    console.log(`Batch: ${totalFiles} total files, ${timeFilteredFiles.length} after time filter, ${extensionFilteredFiles.length} after extension+duplicate filter (${duplicatesRemoved} duplicates removed)`);
                    const validLogs = extensionFilteredFiles
                        .map(obj => ({
                        ...obj,
                        fileType: (0, pulseCronCommon_1.getFileType)(obj.Key || ''),
                        fileExtension: (0, pulseCronCommon_1.getFileExtension)(obj.Key || '')
                    }))
                        .sort((a, b) => new Date(b.LastModified).getTime() - new Date(a.LastModified).getTime());
                    // Add filenames and keys to seen sets to prevent duplicates in future batches
                    validLogs.forEach(log => {
                        const key = log.Key || '';
                        const fileName = this.getFileNameFromPath(key);
                        seenFileNames.add(fileName);
                        seenKeys.add(key);
                    });
                    // Update currentBatchFiles to reflect only the deduplicated files
                    this.currentBatchFiles = new Set(validLogs.map(log => (log.Key || '').toLowerCase()));
                    allValidLogs = this.mergeSortedArrays(allValidLogs, validLogs);
                    // Early termination for count-based requests
                    if (this.samplingByCount && targetCount && allValidLogs.length >= targetCount) {
                        allValidLogs = allValidLogs.slice(0, targetCount);
                        foundEnoughForCount = true;
                        console.log(`Early termination: Found ${targetCount} logs after scanning ${totalScanned} objects`);
                        break;
                    }
                }
                continuationToken = response.NextContinuationToken;
                // Log progress for long operations
                if (Date.now() - startTime > 1000) {
                    console.log(`Scanned ${totalScanned} objects, found ${allValidLogs.length} valid logs`);
                }
            } while (continuationToken && !foundEnoughForCount);
            // Phase 2: Calculate final selection with randomized sampling
            const totalValidCount = allValidLogs.length;
            let finalLogs;
            if (this.samplingByCount) {
                const countToRetrieve = Math.min(this.samplingCount, totalValidCount);
                finalLogs = this.getRandomizedSample(allValidLogs, countToRetrieve);
            }
            else { // percentage
                const percentage = Math.max(0, Math.min(100, this.samplingPercentage));
                const countToRetrieve = Math.ceil((percentage / 100) * totalValidCount);
                finalLogs = this.getRandomizedSample(allValidLogs, countToRetrieve);
            }
            return {
                logs: finalLogs,
                totalValidCount,
                totalScanned,
                retrievedCount: finalLogs.length,
                type: this.samplingByCount ? 'count' : 'percentage',
                value: this.samplingByCount ? this.samplingCount : this.samplingPercentage,
                earlyTerminated: foundEnoughForCount
            };
        }
        catch (error) {
            console.error("Error in optimized log retrieval:", error);
            throw error;
        }
    }
    // Merge two sorted arrays (by LastModified desc) efficiently with duplicate protection
    mergeSortedArrays(arr1, arr2) {
        const result = [];
        const seenKeys = new Set(); // Track seen keys to prevent duplicates
        let i = 0, j = 0;
        while (i < arr1.length && j < arr2.length) {
            const arr1Key = arr1[i].Key || '';
            const arr2Key = arr2[j].Key || '';
            if (new Date(arr1[i].LastModified).getTime() >= new Date(arr2[j].LastModified).getTime()) {
                // Add arr1 element if not already seen
                if (!seenKeys.has(arr1Key)) {
                    result.push(arr1[i]);
                    seenKeys.add(arr1Key);
                }
                i++;
            }
            else {
                // Add arr2 element if not already seen
                if (!seenKeys.has(arr2Key)) {
                    result.push(arr2[j]);
                    seenKeys.add(arr2Key);
                }
                j++;
            }
        }
        // Add remaining elements from arr1
        while (i < arr1.length) {
            const key = arr1[i].Key || '';
            if (!seenKeys.has(key)) {
                result.push(arr1[i]);
                seenKeys.add(key);
            }
            i++;
        }
        // Add remaining elements from arr2
        while (j < arr2.length) {
            const key = arr2[j].Key || '';
            if (!seenKeys.has(key)) {
                result.push(arr2[j]);
                seenKeys.add(key);
            }
            j++;
        }
        return result;
    }
    // Optimized parallel content download with concurrency control
    async getLogContentsParallel(logObjects) {
        const downloadPromises = logObjects.map(async (logObj) => {
            return this.semaphore.acquire(async () => {
                try {
                    const command = new client_s3_1.GetObjectCommand({
                        Bucket: this.bucketName,
                        Key: logObj.Key
                    });
                    const response = await this.client.send(command);
                    const fileType = logObj.fileType || (0, pulseCronCommon_1.getFileType)(logObj.Key);
                    const content = fileType === 'chat' ? await response.Body?.transformToString() : await response.Body?.transformToByteArray();
                    // Get transcript content for audio files
                    let transcript = null;
                    if (fileType === 'audio') {
                        transcript = await this.getTranscriptContent(logObj.Key);
                    }
                    return {
                        key: logObj.Key,
                        lastModified: logObj.LastModified,
                        size: logObj.Size,
                        content,
                        fileType,
                        fileExtension: logObj.fileExtension || (0, pulseCronCommon_1.getFileExtension)(logObj.Key),
                        transcript,
                        fullPath: `https://${this.bucketName}.s3.${this.region}.amazonaws.com/${logObj.Key}`,
                        error: undefined
                    };
                }
                catch (error) {
                    return {
                        key: logObj.Key,
                        lastModified: logObj.LastModified,
                        size: logObj.Size,
                        content: undefined,
                        fileType: logObj.fileType || (0, pulseCronCommon_1.getFileType)(logObj.Key),
                        fileExtension: logObj.fileExtension || (0, pulseCronCommon_1.getFileExtension)(logObj.Key),
                        transcript: null,
                        fullPath: `https://${this.bucketName}.s3.${this.region}.amazonaws.com/${logObj.Key}`,
                        error: error.message
                    };
                }
            });
        });
        return Promise.all(downloadPromises);
    }
    async retrieveLogs() {
        const startTime = Date.now();
        // Create unique lock key for this specific operation
        const lockKey = `${this.applicationId}:${this.bucketName}:${this.prefix}:${this.startTime.getTime()}:${this.endTime.getTime()}`;
        // Check if same operation is already running
        if (PulseCronAWS.GLOBAL_PROCESSING_LOCK.has(lockKey)) {
            console.log(`Operation already running for key: ${lockKey}, waiting for completion...`);
            return await PulseCronAWS.GLOBAL_PROCESSING_LOCK.get(lockKey);
        }
        // Create and store the operation promise
        const operationPromise = this.executeRetrieveLogs(startTime);
        PulseCronAWS.GLOBAL_PROCESSING_LOCK.set(lockKey, operationPromise);
        try {
            const result = await operationPromise;
            return result;
        }
        finally {
            // Clean up the lock
            PulseCronAWS.GLOBAL_PROCESSING_LOCK.delete(lockKey);
        }
    }
    async executeRetrieveLogs(startTime) {
        console.log(`Starting optimized log retrieval...`);
        const metadataResult = await this.getLogs();
        const metadataTime = Date.now() - startTime;
        console.log(`Metadata retrieval completed in ${metadataTime}ms`);
        console.log(`Found ${metadataResult.totalValidCount} valid logs, retrieving ${metadataResult.retrievedCount}`);
        // if (metadataResult.logs.length > 0) {
        //   const contentStartTime = Date.now();
        //   const logContents = await this.getLogContentsParallel(metadataResult.logs);
        //   const contentTime = Date.now() - contentStartTime;
        //   console.log(`Content download completed in ${contentTime}ms`);
        //   console.log(`Total execution time: ${Date.now() - startTime}ms`);
        //   return {
        //     ...metadataResult,
        //     logContents,
        //     performance: {
        //       metadataTime,
        //       contentTime,
        //       totalTime: Date.now() - startTime,
        //       avgDownloadTime: contentTime / logContents.length
        //     }
        //   };
        // }
        // Convert metadata logs to logContents format without downloading content
        const logContents = (metadataResult ?? {})?.logs?.map(logObj => ({
            key: logObj.Key,
            lastModified: logObj.LastModified,
            size: logObj.Size,
            content: undefined,
            fileType: logObj.fileType || (0, pulseCronCommon_1.getFileType)(logObj.Key),
            fileExtension: logObj.fileExtension || (0, pulseCronCommon_1.getFileExtension)(logObj.Key),
            transcript: null,
            fullPath: `https://${this.bucketName}.s3.${this.region}.amazonaws.com/${logObj.Key}`,
            error: undefined
        }));
        console.log(`Metadata processing completed in ${Date.now() - startTime}ms`);
        return {
            ...metadataResult,
            logContents,
            performance: {
                metadataTime,
                contentTime: 0,
                totalTime: Date.now() - startTime,
                avgDownloadTime: 0
            }
        };
    }
    // Memory-efficient streaming for large logs
    async getLogContentsStreaming(logObjects, maxContentSize = 1024 * 1024) {
        const results = [];
        for (const logObj of logObjects) {
            try {
                if (logObj.Size > maxContentSize) {
                    // For large files, get only first part
                    const command = new client_s3_1.GetObjectCommand({
                        Bucket: this.bucketName,
                        Key: logObj.Key,
                        Range: `bytes=0-${maxContentSize - 1}`
                    });
                    const response = await this.client.send(command);
                    const content = await response.Body?.transformToString();
                    const fileType = logObj.fileType || (0, pulseCronCommon_1.getFileType)(logObj.Key);
                    // Get transcript content for audio files
                    let transcript = null;
                    if (fileType === 'audio') {
                        transcript = await this.getTranscriptContent(logObj.Key);
                    }
                    results.push({
                        key: logObj.Key,
                        lastModified: logObj.LastModified,
                        size: logObj.Size,
                        content,
                        fileType,
                        fileExtension: logObj.fileExtension || (0, pulseCronCommon_1.getFileExtension)(logObj.Key),
                        transcript,
                        truncated: true,
                        truncatedSize: maxContentSize
                    });
                }
                else {
                    // Small files: get full content
                    const command = new client_s3_1.GetObjectCommand({
                        Bucket: this.bucketName,
                        Key: logObj.Key
                    });
                    const response = await this.client.send(command);
                    const content = await response.Body?.transformToString();
                    const fileType = logObj.fileType || (0, pulseCronCommon_1.getFileType)(logObj.Key);
                    // Get transcript content for audio files
                    let transcript = null;
                    if (fileType === 'audio') {
                        transcript = await this.getTranscriptContent(logObj.Key);
                    }
                    results.push({
                        key: logObj.Key,
                        lastModified: logObj.LastModified,
                        size: logObj.Size,
                        content,
                        fileType,
                        fileExtension: logObj.fileExtension || (0, pulseCronCommon_1.getFileExtension)(logObj.Key),
                        transcript,
                        truncated: false
                    });
                }
            }
            catch (error) {
                results.push({
                    key: logObj.Key,
                    lastModified: logObj.LastModified,
                    size: logObj.Size,
                    content: undefined,
                    fileType: logObj.fileType || (0, pulseCronCommon_1.getFileType)(logObj.Key),
                    fileExtension: logObj.fileExtension || (0, pulseCronCommon_1.getFileExtension)(logObj.Key),
                    transcript: null,
                    error: error.message
                });
            }
        }
        return results;
    }
    // Get content for a specific file by name
    async getFileContent({ fileName, }) {
        try {
            // Determine file type based on extension
            const fileType = (0, pulseCronCommon_1.getFileType)(fileName);
            const fileExtension = (0, pulseCronCommon_1.getFileExtension)(fileName);
            // Get the file content
            const command = new client_s3_1.GetObjectCommand({
                Bucket: this.bucketName,
                Key: fileName
            });
            const response = await this.client.send(command);
            // Get content based on file type
            let content;
            if (fileType === 'chat') {
                content = await response.Body?.transformToString() || '';
            }
            else {
                content = await response.Body?.transformToByteArray() || new Uint8Array();
            }
            // Get transcript content for audio files
            let transcript = null;
            if (fileType === 'audio') {
                transcript = await this.getTranscriptContent(fileName);
            }
            return {
                key: fileName,
                lastModified: response.LastModified,
                size: response.ContentLength || 0,
                content,
                fileType,
                fileExtension,
                transcript,
                fullPath: `https://${this.bucketName}.s3.${this.region}.amazonaws.com/${fileName}`,
                error: undefined
            };
        }
        catch (error) {
            return {
                key: fileName,
                lastModified: undefined,
                size: 0,
                content: undefined,
                fileType: (0, pulseCronCommon_1.getFileType)(fileName),
                fileExtension: (0, pulseCronCommon_1.getFileExtension)(fileName),
                transcript: null,
                fullPath: `https://${this.bucketName}.s3.${this.region}.amazonaws.com/${fileName}`,
                error: error.message
            };
        }
    }
    // Get content for a specific file by name
    async getFileMetadata({ fileName, }) {
        try {
            // Determine file type based on extension
            const fileType = (0, pulseCronCommon_1.getFileType)(fileName);
            const fileExtension = (0, pulseCronCommon_1.getFileExtension)(fileName);
            return {
                key: fileName,
                lastModified: undefined,
                size: undefined,
                content: undefined,
                fileType,
                fileExtension,
                transcript: null,
                fullPath: `https://${this.bucketName}.s3.${this.region}.amazonaws.com/${fileName}`,
                error: undefined
            };
        }
        catch (error) {
            return {
                key: fileName,
                lastModified: undefined,
                size: 0,
                content: undefined,
                fileType: (0, pulseCronCommon_1.getFileType)(fileName),
                fileExtension: (0, pulseCronCommon_1.getFileExtension)(fileName),
                transcript: null,
                fullPath: `https://${this.bucketName}.s3.${this.region}.amazonaws.com/${fileName}`,
                error: error.message
            };
        }
    }
}
exports.PulseCronAWS = PulseCronAWS;
PulseCronAWS.GLOBAL_PROCESSING_LOCK = new Map(); // Global lock for concurrent protection
//# sourceMappingURL=pulseCronAWS.js.map