"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.PulseCronAzure = void 0;
const storage_blob_1 = require("@azure/storage-blob");
const identity_1 = require("@azure/identity");
const date_fns_1 = require("date-fns");
const semaphore_1 = require("../../../util/semaphore");
const pulseCronCommon_1 = require("./pulseCronCommon");
class PulseCronAzure {
    constructor({ accountName, containerName, connectionString, credential, prefix, maxConcurrentDownloads, maxKeys, startTime, endTime, earlyTermination, samplingByCount, samplingByPercentage, samplingCount, samplingPercentage, validExtensions = Object.values(pulseCronCommon_1.PULSE_CRON_EXTENSION_TYPES).flat(), applicationId }) {
        this.currentBatchFiles = null;
        this.semaphore = new semaphore_1.Semaphore(10);
        this.accountName = accountName;
        this.containerName = containerName;
        this.prefix = prefix;
        this.maxConcurrentDownloads = maxConcurrentDownloads;
        this.maxKeys = maxKeys;
        this.startTime = typeof startTime === 'string' ? (0, date_fns_1.parseISO)(startTime.replace(' ', 'T') + 'Z') : startTime;
        this.endTime = typeof endTime === 'string' ? (0, date_fns_1.parseISO)(endTime.replace(' ', 'T') + 'Z') : endTime;
        this.earlyTermination = earlyTermination;
        this.samplingByCount = samplingByCount;
        this.samplingByPercentage = samplingByPercentage;
        this.samplingCount = samplingCount;
        this.samplingPercentage = samplingPercentage;
        this.validExtensions = validExtensions;
        this.applicationId = applicationId;
        // Initialize Azure Storage client
        if (connectionString) {
            this.client = storage_blob_1.BlobServiceClient.fromConnectionString(connectionString);
        }
        else if (credential) {
            this.client = new storage_blob_1.BlobServiceClient(`https://${accountName}.blob.core.windows.net`, credential);
        }
        else {
            // Use DefaultAzureCredential for managed identity or environment variables
            this.client = new storage_blob_1.BlobServiceClient(`https://${accountName}.blob.core.windows.net`, new identity_1.DefaultAzureCredential());
        }
        this.containerClient = this.client.getContainerClient(containerName);
    }
    // Check if file is a transcript file for an audio file
    isTranscriptFile(name) {
        const lowerName = name.toLowerCase();
        const transcriptExtensions = ['.json', '.log', '.txt'];
        // Check if it ends with transcript extension
        const hasTranscriptExt = transcriptExtensions.some(ext => lowerName.endsWith(ext));
        if (!hasTranscriptExt) {
            return false;
        }
        // Check if there's a corresponding audio file with same base name
        const baseName = (0, pulseCronCommon_1.getBaseFileName)(name);
        const audioExtensions = ['.wav', '.mp3', '.ogg', '.flac', '.pcm16'];
        return audioExtensions.some(audioExt => {
            const potentialAudioFile = baseName + audioExt;
            return this.fileExists(potentialAudioFile);
        });
    }
    // Check if a file exists in the current batch (this will be populated during processing)
    fileExists(fileName) {
        // This will be implemented to check against the current batch of files
        return this.currentBatchFiles?.has(fileName.toLowerCase()) || false;
    }
    // Get transcript content for an audio file
    async getTranscriptContent(audioFileName) {
        const baseName = (0, pulseCronCommon_1.getBaseFileName)(audioFileName);
        const transcriptExtensions = ['.json', '.log', '.txt'];
        for (const ext of transcriptExtensions) {
            const transcriptName = baseName + ext;
            try {
                const blobClient = this.containerClient.getBlobClient(transcriptName);
                const downloadResponse = await blobClient.download();
                const content = await this.streamToString(downloadResponse.readableStreamBody);
                return content || null;
            }
            catch {
                // Transcript file doesn't exist, try next extension
                continue;
            }
        }
        return null;
    }
    // Perform randomized sampling with guaranteed uniqueness
    getRandomizedSample(logs, targetCount) {
        if (targetCount >= logs.length) {
            return logs; // Return all logs if target count is greater than or equal to total
        }
        const selectedLogs = [];
        const selectedIndices = new Set(); // Track selected indices for O(1) lookup
        for (let i = 0; i < targetCount; i++) {
            let attempts = 0;
            const maxAttempts = logs.length * 2; // Prevent infinite loops
            while (attempts < maxAttempts) {
                const index = Math.floor(Math.random() * logs.length);
                if (!selectedIndices.has(index)) {
                    selectedLogs.push(logs[index]);
                    selectedIndices.add(index);
                    break;
                }
                attempts++;
            }
            // If we couldn't find a unique index, break to avoid infinite loop
            if (attempts >= maxAttempts) {
                console.warn(`Could not find unique log after ${maxAttempts} attempts. Selected ${selectedLogs.length} logs instead of ${targetCount}`);
                break;
            }
        }
        return selectedLogs;
    }
    // Get filename from full path (extract just the filename, not the path)
    getFileNameFromPath(fullPath) {
        const lastSlashIndex = fullPath.lastIndexOf('/');
        return lastSlashIndex === -1 ? fullPath : fullPath.substring(lastSlashIndex + 1);
    }
    // Get logs with early termination and smart pagination
    async getLogs() {
        const targetCount = this.samplingByCount ? this.samplingCount : null;
        console.log(`Filtering logs: startTime=${this.startTime.toISOString()}, endTime=${this.endTime.toISOString()}`);
        console.log(`Time range: ${this.endTime.getTime() - this.startTime.getTime()}ms (${(this.endTime.getTime() - this.startTime.getTime()) / (1000 * 60 * 60)} hours)`);
        let allValidLogs = [];
        const allTimeFilteredFiles = [];
        const seenFileNames = new Set(); // Track seen filenames for deduplication
        const seenBlobNames = new Set(); // Track seen blob names to prevent iterator duplicates
        let totalScanned = 0;
        let foundEnoughForCount = false;
        try {
            // Phase 1: Fast metadata collection with early termination
            // Azure Blob Storage doesn't support time-based filtering in listBlobsFlat
            // So we need to iterate through all blobs and filter by time
            // We'll use pagination to control memory usage and provide progress updates
            const listOptions = {
                prefix: this.prefix,
                maxPageSize: this.maxKeys
            };
            const iterator = this.containerClient.listBlobsFlat(listOptions);
            let pageCount = 0;
            let batchStartTime = Date.now();
            for await (const blob of iterator) {
                totalScanned++;
                pageCount++;
                // Check for iterator duplicates first (Azure API can return same blob twice)
                if (seenBlobNames.has(blob.name)) {
                    console.log(`Skipping duplicate blob from iterator: ${blob.name}`);
                    continue;
                }
                seenBlobNames.add(blob.name);
                // Filter by time first
                const lastModified = blob.properties.lastModified;
                if (lastModified && lastModified >= this.startTime && lastModified <= this.endTime) {
                    allTimeFilteredFiles.push(blob);
                }
                // Log progress for long operations and provide batch processing insights
                if (pageCount % 1000 === 0) {
                    const batchTime = Date.now() - batchStartTime;
                    const rate = pageCount / (batchTime / 1000); // blobs per second
                    console.log(`Scanned ${totalScanned} objects, found ${allTimeFilteredFiles.length} time-filtered files, rate: ${rate.toFixed(2)} blobs/sec`);
                    batchStartTime = Date.now();
                }
            }
            // First, populate currentBatchFiles for transcript detection
            this.currentBatchFiles = new Set(allTimeFilteredFiles.map(blob => blob.name.toLowerCase()));
            // Now filter by extension, exclude transcript files, and check for duplicates
            for (const blob of allTimeFilteredFiles) {
                if ((0, pulseCronCommon_1.hasValidExtension)(blob.name, this.validExtensions) && !this.isTranscriptFile(blob.name)) {
                    // Check for duplicates based on filename
                    const fileName = this.getFileNameFromPath(blob.name);
                    if (seenFileNames.has(fileName)) {
                        continue; // Skip duplicate filename
                    }
                    const validLog = {
                        Key: blob.name,
                        LastModified: blob.properties.lastModified,
                        Size: blob.properties.contentLength || 0,
                        fileType: (0, pulseCronCommon_1.getFileType)(blob.name),
                        fileExtension: (0, pulseCronCommon_1.getFileExtension)(blob.name)
                    };
                    allValidLogs.push(validLog);
                    seenFileNames.add(fileName); // Track this filename
                    // Early termination for count-based requests
                    if (this.samplingByCount && targetCount && allValidLogs.length >= targetCount) {
                        allValidLogs = allValidLogs.slice(0, targetCount);
                        foundEnoughForCount = true;
                        console.log(`Early termination: Found ${targetCount} logs after scanning ${totalScanned} objects`);
                        break;
                    }
                }
            }
            // Sort by LastModified desc
            allValidLogs.sort((a, b) => new Date(b.LastModified).getTime() - new Date(a.LastModified).getTime());
            // Update currentBatchFiles to reflect only the deduplicated files
            this.currentBatchFiles = new Set(allValidLogs.map(log => (log.Key || '').toLowerCase()));
            console.log(`Azure processing complete: ${allTimeFilteredFiles.length} time-filtered files, ${allValidLogs.length} unique valid logs after deduplication`);
            // Phase 2: Calculate final selection with randomized sampling
            const totalValidCount = allValidLogs.length;
            let finalLogs;
            if (this.samplingByCount) {
                const countToRetrieve = Math.min(this.samplingCount, totalValidCount);
                finalLogs = this.getRandomizedSample(allValidLogs, countToRetrieve);
            }
            else { // percentage
                const percentage = Math.max(0, Math.min(100, this.samplingPercentage));
                const countToRetrieve = Math.ceil((percentage / 100) * totalValidCount);
                finalLogs = this.getRandomizedSample(allValidLogs, countToRetrieve);
            }
            return {
                logs: finalLogs,
                totalValidCount,
                totalScanned,
                retrievedCount: finalLogs.length,
                type: this.samplingByCount ? 'count' : 'percentage',
                value: this.samplingByCount ? this.samplingCount : this.samplingPercentage,
                earlyTerminated: foundEnoughForCount
            };
        }
        catch (error) {
            console.error("Error in optimized log retrieval:", error);
            throw error;
        }
    }
    // Optimized parallel content download with concurrency control
    async getLogContentsParallel(logObjects) {
        const downloadPromises = logObjects.map(async (logObj) => {
            return this.semaphore.acquire(async () => {
                try {
                    const blobClient = this.containerClient.getBlobClient(logObj.Key);
                    const downloadResponse = await blobClient.download();
                    const fileType = logObj.fileType || (0, pulseCronCommon_1.getFileType)(logObj.Key);
                    let content;
                    if (fileType === 'chat') {
                        content = await this.streamToString(downloadResponse.readableStreamBody);
                    }
                    else {
                        content = await this.streamToByteArray(downloadResponse.readableStreamBody);
                    }
                    // Get transcript content for audio files
                    let transcript = null;
                    if (fileType === 'audio') {
                        transcript = await this.getTranscriptContent(logObj.Key);
                    }
                    return {
                        key: logObj.Key,
                        lastModified: logObj.LastModified,
                        size: logObj.Size,
                        content,
                        fileType,
                        fileExtension: logObj.fileExtension || (0, pulseCronCommon_1.getFileExtension)(logObj.Key),
                        transcript,
                        fullPath: `https://${this.accountName}.blob.core.windows.net/${this.containerName}/${logObj.Key}`,
                        error: undefined
                    };
                }
                catch (error) {
                    return {
                        key: logObj.Key,
                        lastModified: logObj.LastModified,
                        size: logObj.Size,
                        content: undefined,
                        fileType: logObj.fileType || (0, pulseCronCommon_1.getFileType)(logObj.Key),
                        fileExtension: logObj.fileExtension || (0, pulseCronCommon_1.getFileExtension)(logObj.Key),
                        transcript: null,
                        fullPath: `https://${this.accountName}.blob.core.windows.net/${this.containerName}/${logObj.Key}`,
                        error: error.message
                    };
                }
            });
        });
        return Promise.all(downloadPromises);
    }
    // Helper method to convert stream to string
    async streamToString(readableStream) {
        return new Promise((resolve, reject) => {
            const chunks = [];
            readableStream.on('data', (data) => {
                chunks.push(Buffer.from(data));
            });
            readableStream.on('end', () => {
                resolve(Buffer.concat(chunks).toString('utf8'));
            });
            readableStream.on('error', reject);
        });
    }
    // Helper method to convert stream to byte array
    async streamToByteArray(readableStream) {
        return new Promise((resolve, reject) => {
            const chunks = [];
            readableStream.on('data', (data) => {
                chunks.push(Buffer.from(data));
            });
            readableStream.on('end', () => {
                resolve(new Uint8Array(Buffer.concat(chunks)));
            });
            readableStream.on('error', reject);
        });
    }
    async retrieveLogs() {
        const startTime = Date.now();
        // Create unique lock key for this specific operation
        const lockKey = `${this.applicationId}:${this.accountName}:${this.containerName}:${this.prefix}:${this.startTime.getTime()}:${this.endTime.getTime()}`;
        // Check if same operation is already running
        if (PulseCronAzure.GLOBAL_PROCESSING_LOCK.has(lockKey)) {
            console.log(`Operation already running for key: ${lockKey}, waiting for completion...`);
            return await PulseCronAzure.GLOBAL_PROCESSING_LOCK.get(lockKey);
        }
        // Create and store the operation promise
        const operationPromise = this.executeRetrieveLogs(startTime);
        PulseCronAzure.GLOBAL_PROCESSING_LOCK.set(lockKey, operationPromise);
        try {
            const result = await operationPromise;
            return result;
        }
        finally {
            // Clean up the lock
            PulseCronAzure.GLOBAL_PROCESSING_LOCK.delete(lockKey);
        }
    }
    async executeRetrieveLogs(startTime) {
        console.log(`Starting optimized log retrieval...`);
        const metadataResult = await this.getLogs();
        const metadataTime = Date.now() - startTime;
        console.log(`Metadata retrieval completed in ${metadataTime}ms`);
        console.log(`Found ${metadataResult.totalValidCount} valid logs, retrieving ${metadataResult.retrievedCount}`);
        // if (metadataResult.logs.length > 0) {
        //   const contentStartTime = Date.now();
        //   const logContents = await this.getLogContentsParallel(metadataResult.logs);
        //   const contentTime = Date.now() - contentStartTime;
        //   console.log(`Content download completed in ${contentTime}ms`);
        //   console.log(`Total execution time: ${Date.now() - startTime}ms`);
        //   return {
        //     ...metadataResult,
        //     logContents,
        //     performance: {
        //       metadataTime,
        //       contentTime,
        //       totalTime: Date.now() - startTime,
        //       avgDownloadTime: contentTime / logContents.length
        //     }
        //   };
        // }
        // Convert metadata logs to logContents format without downloading content
        const logContents = (metadataResult ?? {})?.logs?.map(logObj => ({
            key: logObj.Key,
            lastModified: logObj.LastModified,
            size: logObj.Size,
            content: undefined,
            fileType: logObj.fileType || (0, pulseCronCommon_1.getFileType)(logObj.Key),
            fileExtension: logObj.fileExtension || (0, pulseCronCommon_1.getFileExtension)(logObj.Key),
            transcript: null,
            fullPath: `https://${this.accountName}.blob.core.windows.net/${this.containerName}/${logObj.Key}`,
            error: undefined
        }));
        console.log(`Metadata processing completed in ${Date.now() - startTime}ms`);
        return {
            ...metadataResult,
            logContents,
            performance: {
                metadataTime,
                contentTime: 0,
                totalTime: Date.now() - startTime,
                avgDownloadTime: 0
            }
        };
    }
    // Memory-efficient streaming for large logs
    async getLogContentsStreaming(logObjects, maxContentSize = 1024 * 1024) {
        const results = [];
        for (const logObj of logObjects) {
            try {
                if (logObj.Size > maxContentSize) {
                    // For large files, get only first part
                    const blobClient = this.containerClient.getBlobClient(logObj.Key);
                    const downloadResponse = await blobClient.download(0, maxContentSize - 1);
                    const content = await this.streamToString(downloadResponse.readableStreamBody);
                    const fileType = logObj.fileType || (0, pulseCronCommon_1.getFileType)(logObj.Key);
                    // Get transcript content for audio files
                    let transcript = null;
                    if (fileType === 'audio') {
                        transcript = await this.getTranscriptContent(logObj.Key);
                    }
                    results.push({
                        key: logObj.Key,
                        lastModified: logObj.LastModified,
                        size: logObj.Size,
                        content,
                        fileType,
                        fileExtension: logObj.fileExtension || (0, pulseCronCommon_1.getFileExtension)(logObj.Key),
                        transcript,
                        truncated: true,
                        truncatedSize: maxContentSize
                    });
                }
                else {
                    // Small files: get full content
                    const blobClient = this.containerClient.getBlobClient(logObj.Key);
                    const downloadResponse = await blobClient.download();
                    const content = await this.streamToString(downloadResponse.readableStreamBody);
                    const fileType = logObj.fileType || (0, pulseCronCommon_1.getFileType)(logObj.Key);
                    // Get transcript content for audio files
                    let transcript = null;
                    if (fileType === 'audio') {
                        transcript = await this.getTranscriptContent(logObj.Key);
                    }
                    results.push({
                        key: logObj.Key,
                        lastModified: logObj.LastModified,
                        size: logObj.Size,
                        content,
                        fileType,
                        fileExtension: logObj.fileExtension || (0, pulseCronCommon_1.getFileExtension)(logObj.Key),
                        transcript,
                        truncated: false
                    });
                }
            }
            catch (error) {
                results.push({
                    key: logObj.Key,
                    lastModified: logObj.LastModified,
                    size: logObj.Size,
                    content: undefined,
                    fileType: logObj.fileType || (0, pulseCronCommon_1.getFileType)(logObj.Key),
                    fileExtension: logObj.fileExtension || (0, pulseCronCommon_1.getFileExtension)(logObj.Key),
                    transcript: null,
                    error: error.message
                });
            }
        }
        return results;
    }
    // Get content for a specific file by name
    async getFileContent({ fileName, }) {
        try {
            // Determine file type based on extension
            const fileType = (0, pulseCronCommon_1.getFileType)(fileName);
            const fileExtension = (0, pulseCronCommon_1.getFileExtension)(fileName);
            // Get the file content
            const blobClient = this.containerClient.getBlobClient(fileName);
            const downloadResponse = await blobClient.download();
            // Get content based on file type
            let content;
            if (fileType === 'chat') {
                content = await this.streamToString(downloadResponse.readableStreamBody);
            }
            else {
                content = await this.streamToByteArray(downloadResponse.readableStreamBody);
            }
            // Get transcript content for audio files
            let transcript = null;
            if (fileType === 'audio') {
                transcript = await this.getTranscriptContent(fileName);
            }
            return {
                key: fileName,
                lastModified: downloadResponse.lastModified,
                size: downloadResponse.contentLength || 0,
                content,
                fileType,
                fileExtension,
                transcript,
                fullPath: `https://${this.accountName}.blob.core.windows.net/${this.containerName}/${fileName}`,
                error: undefined
            };
        }
        catch (error) {
            return {
                key: fileName,
                lastModified: undefined,
                size: 0,
                content: undefined,
                fileType: (0, pulseCronCommon_1.getFileType)(fileName),
                fileExtension: (0, pulseCronCommon_1.getFileExtension)(fileName),
                transcript: null,
                fullPath: `https://${this.accountName}.blob.core.windows.net/${this.containerName}/${fileName}`,
                error: error.message
            };
        }
    }
    // Get content for a specific file by name
    async getFileMetadata({ fileName, }) {
        try {
            // Determine file type based on extension
            const fileType = (0, pulseCronCommon_1.getFileType)(fileName);
            const fileExtension = (0, pulseCronCommon_1.getFileExtension)(fileName);
            return {
                key: fileName,
                lastModified: undefined,
                size: undefined,
                content: undefined,
                fileType,
                fileExtension,
                transcript: null,
                fullPath: `https://${this.accountName}.blob.core.windows.net/${this.containerName}/${fileName}`,
                error: undefined
            };
        }
        catch (error) {
            return {
                key: fileName,
                lastModified: undefined,
                size: 0,
                content: undefined,
                fileType: (0, pulseCronCommon_1.getFileType)(fileName),
                fileExtension: (0, pulseCronCommon_1.getFileExtension)(fileName),
                transcript: null,
                fullPath: `https://${this.accountName}.blob.core.windows.net/${this.containerName}/${fileName}`,
                error: error.message
            };
        }
    }
}
exports.PulseCronAzure = PulseCronAzure;
PulseCronAzure.GLOBAL_PROCESSING_LOCK = new Map(); // Global lock for concurrent protection
//# sourceMappingURL=pulseCronAzure.js.map