"use strict";
/**
 * Advanced Metrics Collection and Aggregation System for DataPilot
 * Collects, processes, and aggregates performance metrics for production monitoring
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.MetricsUtils = exports.globalMetricsCollector = exports.MetricsCollector = exports.MetricType = void 0;
const events_1 = require("events");
const perf_hooks_1 = require("perf_hooks");
const logger_1 = require("../utils/logger");
const performance_monitor_1 = require("../core/performance-monitor");
const memory_manager_1 = require("../utils/memory-manager");
const error_handler_1 = require("../utils/error-handler");
var MetricType;
(function (MetricType) {
    MetricType["COUNTER"] = "counter";
    MetricType["GAUGE"] = "gauge";
    MetricType["HISTOGRAM"] = "histogram";
    MetricType["SUMMARY"] = "summary";
})(MetricType || (exports.MetricType = MetricType = {}));
/**
 * Production metrics collection and aggregation system
 */
class MetricsCollector extends events_1.EventEmitter {
    static instance;
    metrics = new Map();
    configs = new Map();
    aggregations = new Map();
    isCollecting = false;
    collectionInterval;
    aggregationInterval;
    startTime = Date.now();
    sectionMetrics = new Map();
    // Request tracking
    requestCount = 0;
    errorCount = 0;
    responseTimes = [];
    dataProcessingStats = {
        rowsProcessed: 0,
        filesProcessed: 0,
        bytesProcessed: 0,
    };
    constructor() {
        super();
        this.initializeDefaultMetrics();
        this.initializeSectionMetrics();
    }
    static getInstance() {
        if (!MetricsCollector.instance) {
            MetricsCollector.instance = new MetricsCollector();
        }
        return MetricsCollector.instance;
    }
    /**
     * Register a new metric
     */
    registerMetric(config) {
        this.configs.set(config.name, {
            retention: 24 * 60 * 60 * 1000, // 24 hours default
            aggregationWindow: 5 * 60 * 1000, // 5 minutes default
            ...config,
        });
        this.metrics.set(config.name, {
            metric: config.name,
            values: [],
            retention: config.retention || 24 * 60 * 60 * 1000,
        });
        logger_1.logger.debug(`Registered metric: ${config.name} (${config.type})`, {
            operation: 'registerMetric',
        });
    }
    /**
     * Record a metric value
     */
    recordMetric(name, value, labels, timestamp) {
        const timeSeries = this.metrics.get(name);
        if (!timeSeries) {
            logger_1.logger.warn(`Metric '${name}' not registered`, { operation: 'recordMetric' });
            return;
        }
        const metricValue = {
            value,
            timestamp: timestamp || Date.now(),
            labels,
        };
        timeSeries.values.push(metricValue);
        this.emit('metricRecorded', name, metricValue);
        // Clean up old values
        this.cleanupTimeSeries(timeSeries);
    }
    /**
     * Increment a counter metric
     */
    incrementCounter(name, amount = 1, labels) {
        const config = this.configs.get(name);
        if (!config || config.type !== MetricType.COUNTER) {
            logger_1.logger.warn(`Counter '${name}' not found or wrong type`, { operation: 'incrementCounter' });
            return;
        }
        this.recordMetric(name, amount, labels);
    }
    /**
     * Set a gauge metric
     */
    setGauge(name, value, labels) {
        const config = this.configs.get(name);
        if (!config || config.type !== MetricType.GAUGE) {
            logger_1.logger.warn(`Gauge '${name}' not found or wrong type`, { operation: 'setGauge' });
            return;
        }
        this.recordMetric(name, value, labels);
    }
    /**
     * Record a histogram value
     */
    recordHistogram(name, value, labels) {
        const config = this.configs.get(name);
        if (!config || config.type !== MetricType.HISTOGRAM) {
            logger_1.logger.warn(`Histogram '${name}' not found or wrong type`, { operation: 'recordHistogram' });
            return;
        }
        this.recordMetric(name, value, labels);
    }
    /**
     * Record timing information
     */
    recordTiming(name, startTime, labels) {
        const duration = perf_hooks_1.performance.now() - startTime;
        this.recordHistogram(name, duration, labels);
    }
    /**
     * Start timing operation
     */
    startTimer(name) {
        const startTime = perf_hooks_1.performance.now();
        return (labels) => {
            this.recordTiming(name, startTime, labels);
        };
    }
    /**
     * Record section execution metrics
     */
    recordSectionExecution(sectionName, duration, success, context) {
        const metrics = this.sectionMetrics.get(sectionName) || {
            executionCount: 0,
            totalDuration: 0,
            avgDuration: 0,
            maxDuration: 0,
            errorCount: 0,
            successRate: 100,
            lastExecution: 0,
        };
        metrics.executionCount++;
        metrics.totalDuration += duration;
        metrics.avgDuration = metrics.totalDuration / metrics.executionCount;
        metrics.maxDuration = Math.max(metrics.maxDuration, duration);
        metrics.lastExecution = Date.now();
        if (!success) {
            metrics.errorCount++;
        }
        metrics.successRate =
            ((metrics.executionCount - metrics.errorCount) / metrics.executionCount) * 100;
        this.sectionMetrics.set(sectionName, metrics);
        // Record as standard metrics
        this.recordHistogram('section_duration_ms', duration, { section: sectionName });
        this.incrementCounter('section_executions_total', 1, {
            section: sectionName,
            success: success.toString(),
        });
        logger_1.logger.debug(`Recorded section execution: ${sectionName} (${duration.toFixed(2)}ms, success: ${success})`, {
            ...context,
            operation: 'recordSectionExecution',
            section: sectionName,
            duration,
        });
    }
    /**
     * Record data processing metrics
     */
    recordDataProcessing(rowsProcessed, bytesProcessed, fileProcessed = false) {
        this.dataProcessingStats.rowsProcessed += rowsProcessed;
        this.dataProcessingStats.bytesProcessed += bytesProcessed;
        if (fileProcessed) {
            this.dataProcessingStats.filesProcessed++;
        }
        // Record as standard metrics
        this.incrementCounter('rows_processed_total', rowsProcessed);
        this.incrementCounter('bytes_processed_total', bytesProcessed);
        if (fileProcessed) {
            this.incrementCounter('files_processed_total', 1);
        }
        // Calculate processing rate (rows per second)
        const uptimeSeconds = (Date.now() - this.startTime) / 1000;
        const processingRate = this.dataProcessingStats.rowsProcessed / uptimeSeconds;
        this.setGauge('processing_rate_rows_per_second', processingRate);
    }
    /**
     * Record request metrics
     */
    recordRequest(responseTime, success) {
        this.requestCount++;
        this.responseTimes.push(responseTime);
        if (!success) {
            this.errorCount++;
        }
        // Keep only recent response times
        if (this.responseTimes.length > 1000) {
            this.responseTimes = this.responseTimes.slice(-1000);
        }
        // Record as standard metrics
        this.incrementCounter('requests_total', 1, { success: success.toString() });
        this.recordHistogram('request_duration_ms', responseTime);
        const errorRate = (this.errorCount / this.requestCount) * 100;
        this.setGauge('error_rate_percent', errorRate);
    }
    /**
     * Get current system metrics
     */
    async getSystemMetrics() {
        const memoryUsage = process.memoryUsage();
        const uptimeMs = Date.now() - this.startTime;
        const uptimeSeconds = uptimeMs / 1000;
        // Calculate averages
        const avgResponseTime = this.responseTimes.length > 0
            ? this.responseTimes.reduce((a, b) => a + b, 0) / this.responseTimes.length
            : 0;
        const p95ResponseTime = this.calculatePercentile(this.responseTimes, 0.95);
        const p99ResponseTime = this.calculatePercentile(this.responseTimes, 0.99);
        const requestsPerSecond = uptimeSeconds > 0 ? this.requestCount / uptimeSeconds : 0;
        const errorRate = this.requestCount > 0 ? (this.errorCount / this.requestCount) * 100 : 0;
        const processingRate = uptimeSeconds > 0 ? this.dataProcessingStats.rowsProcessed / uptimeSeconds : 0;
        return {
            timestamp: Date.now(),
            system: {
                uptime: uptimeMs,
                memory: {
                    total: memoryUsage.heapTotal + memoryUsage.external,
                    used: memoryUsage.heapUsed,
                    free: memoryUsage.heapTotal - memoryUsage.heapUsed,
                    heapTotal: memoryUsage.heapTotal,
                    heapUsed: memoryUsage.heapUsed,
                    external: memoryUsage.external,
                    rss: memoryUsage.rss,
                },
                cpu: {
                    usage: process.cpuUsage().user / 1000000, // Convert to seconds
                    loadAverage: [],
                },
                process: {
                    pid: process.pid,
                    ppid: process.ppid || 0,
                    platform: process.platform,
                    arch: process.arch,
                    version: process.version,
                },
            },
            application: {
                requestsTotal: this.requestCount,
                requestsPerSecond,
                responseTimeMs: {
                    avg: avgResponseTime,
                    p95: p95ResponseTime,
                    p99: p99ResponseTime,
                },
                errorsTotal: this.errorCount,
                errorRate,
                dataProcessing: {
                    rowsProcessed: this.dataProcessingStats.rowsProcessed,
                    filesProcessed: this.dataProcessingStats.filesProcessed,
                    bytesProcessed: this.dataProcessingStats.bytesProcessed,
                    processingRate,
                },
                sections: {
                    section1: this.sectionMetrics.get('section1') || this.getDefaultSectionMetrics(),
                    section2: this.sectionMetrics.get('section2') || this.getDefaultSectionMetrics(),
                    section3: this.sectionMetrics.get('section3') || this.getDefaultSectionMetrics(),
                    section4: this.sectionMetrics.get('section4') || this.getDefaultSectionMetrics(),
                    section5: this.sectionMetrics.get('section5') || this.getDefaultSectionMetrics(),
                    section6: this.sectionMetrics.get('section6') || this.getDefaultSectionMetrics(),
                },
            },
        };
    }
    /**
     * Get metric summary with aggregations
     */
    getMetricSummary(name) {
        const timeSeries = this.metrics.get(name);
        const config = this.configs.get(name);
        if (!timeSeries || !config) {
            return undefined;
        }
        const values = timeSeries.values.map((v) => v.value);
        if (values.length === 0) {
            return undefined;
        }
        const sortedValues = [...values].sort((a, b) => a - b);
        const sum = values.reduce((a, b) => a + b, 0);
        const avg = sum / values.length;
        return {
            name,
            type: config.type,
            description: config.description,
            unit: config.unit,
            currentValue: values[values.length - 1],
            aggregations: {
                min: Math.min(...values),
                max: Math.max(...values),
                avg,
                p50: this.calculatePercentile(sortedValues, 0.5),
                p95: this.calculatePercentile(sortedValues, 0.95),
                p99: this.calculatePercentile(sortedValues, 0.99),
                sum,
                count: values.length,
            },
            lastUpdated: timeSeries.values[timeSeries.values.length - 1]?.timestamp || 0,
        };
    }
    /**
     * Get all metric summaries
     */
    getAllMetricSummaries() {
        return Array.from(this.configs.keys())
            .map((name) => this.getMetricSummary(name))
            .filter((summary) => summary !== undefined);
    }
    /**
     * Export metrics in Prometheus format
     */
    exportPrometheusMetrics() {
        const lines = [];
        const timestamp = Date.now();
        for (const [name, config] of this.configs.entries()) {
            const summary = this.getMetricSummary(name);
            if (!summary)
                continue;
            lines.push(`# HELP datapilot_${name} ${config.description}`);
            lines.push(`# TYPE datapilot_${name} ${config.type}`);
            switch (config.type) {
                case MetricType.COUNTER:
                case MetricType.GAUGE:
                    lines.push(`datapilot_${name} ${summary.currentValue} ${timestamp}`);
                    break;
                case MetricType.HISTOGRAM:
                    // Export histogram buckets and summary statistics
                    lines.push(`datapilot_${name}_count ${summary.aggregations.count} ${timestamp}`);
                    lines.push(`datapilot_${name}_sum ${summary.aggregations.sum} ${timestamp}`);
                    lines.push(`datapilot_${name}_avg ${summary.aggregations.avg} ${timestamp}`);
                    lines.push(`datapilot_${name}_p95 ${summary.aggregations.p95} ${timestamp}`);
                    lines.push(`datapilot_${name}_p99 ${summary.aggregations.p99} ${timestamp}`);
                    break;
                case MetricType.SUMMARY:
                    lines.push(`datapilot_${name}_count ${summary.aggregations.count} ${timestamp}`);
                    lines.push(`datapilot_${name}_sum ${summary.aggregations.sum} ${timestamp}`);
                    lines.push(`datapilot_${name}{quantile="0.5"} ${summary.aggregations.p50} ${timestamp}`);
                    lines.push(`datapilot_${name}{quantile="0.95"} ${summary.aggregations.p95} ${timestamp}`);
                    lines.push(`datapilot_${name}{quantile="0.99"} ${summary.aggregations.p99} ${timestamp}`);
                    break;
            }
        }
        return lines.join('\n') + '\n';
    }
    /**
     * Export metrics in JSON format
     */
    exportJSONMetrics() {
        return {
            timestamp: Date.now(),
            uptime: Date.now() - this.startTime,
            metrics: this.getAllMetricSummaries(),
            system: {
                requests: {
                    total: this.requestCount,
                    errors: this.errorCount,
                    errorRate: this.requestCount > 0 ? (this.errorCount / this.requestCount) * 100 : 0,
                },
                dataProcessing: this.dataProcessingStats,
                sections: Object.fromEntries(this.sectionMetrics.entries()),
            },
        };
    }
    /**
     * Start automatic metrics collection
     */
    startCollection(intervalMs = 30000) {
        if (this.isCollecting) {
            logger_1.logger.warn('Metrics collection already started');
            return;
        }
        this.isCollecting = true;
        logger_1.logger.info(`Starting metrics collection with ${intervalMs}ms interval`);
        // Collect system metrics periodically
        this.collectionInterval = setInterval(async () => {
            try {
                await this.collectSystemMetrics();
            }
            catch (error) {
                logger_1.logger.error(`Error collecting system metrics: ${error instanceof Error ? error.message : 'Unknown error'}`, {
                    operation: 'collectSystemMetrics',
                    error: error instanceof Error ? error.stack : String(error),
                });
            }
        }, intervalMs);
        // Aggregate metrics less frequently
        this.aggregationInterval = setInterval(() => {
            try {
                this.aggregateMetrics();
            }
            catch (error) {
                logger_1.logger.error(`Error aggregating metrics: ${error instanceof Error ? error.message : 'Unknown error'}`, {
                    operation: 'aggregateMetrics',
                    error: error instanceof Error ? error.stack : String(error),
                });
            }
        }, intervalMs * 2);
    }
    /**
     * Stop metrics collection
     */
    stopCollection() {
        if (!this.isCollecting) {
            return;
        }
        this.isCollecting = false;
        if (this.collectionInterval) {
            clearInterval(this.collectionInterval);
            this.collectionInterval = undefined;
        }
        if (this.aggregationInterval) {
            clearInterval(this.aggregationInterval);
            this.aggregationInterval = undefined;
        }
        logger_1.logger.info('Metrics collection stopped');
    }
    /**
     * Reset all metrics
     */
    reset() {
        this.metrics.clear();
        this.aggregations.clear();
        this.requestCount = 0;
        this.errorCount = 0;
        this.responseTimes = [];
        this.dataProcessingStats = {
            rowsProcessed: 0,
            filesProcessed: 0,
            bytesProcessed: 0,
        };
        this.sectionMetrics.clear();
        this.startTime = Date.now();
        this.initializeSectionMetrics();
        logger_1.logger.info('Metrics reset');
    }
    /**
     * Dispose of the metrics collector
     */
    dispose() {
        this.stopCollection();
        this.removeAllListeners();
        this.metrics.clear();
        this.configs.clear();
        this.aggregations.clear();
    }
    async collectSystemMetrics() {
        // Collect memory metrics
        const memoryUsage = memory_manager_1.globalMemoryManager.checkMemoryUsage();
        this.setGauge('memory_heap_used_bytes', memoryUsage.heapUsed);
        this.setGauge('memory_heap_total_bytes', memoryUsage.heapTotal);
        this.setGauge('memory_external_bytes', memoryUsage.external);
        this.setGauge('memory_rss_bytes', memoryUsage.rss);
        // Collect performance metrics
        const performanceMonitor = (0, performance_monitor_1.getPerformanceMonitor)();
        const performanceSummary = performanceMonitor.getPerformanceSummary();
        if (performanceSummary.currentMetrics) {
            this.setGauge('cpu_usage_seconds', performanceSummary.currentMetrics.cpuTime || 0);
            this.setGauge('processing_rate_rows_per_second', performanceSummary.currentMetrics.processingRate || 0);
            this.setGauge('throughput_operations_per_second', performanceSummary.currentMetrics.throughput || 0);
        }
        // Collect error metrics
        const errorStats = error_handler_1.globalErrorHandler.getStats();
        this.setGauge('errors_total', errorStats.totalErrors);
        this.setGauge('critical_errors_total', errorStats.criticalErrors);
        this.setGauge('recovered_errors_total', errorStats.recoveredErrors);
        this.emit('systemMetricsCollected');
    }
    aggregateMetrics() {
        for (const [name, config] of this.configs.entries()) {
            const summary = this.getMetricSummary(name);
            if (summary) {
                this.aggregations.set(name, summary);
            }
        }
        this.emit('metricsAggregated');
    }
    cleanupTimeSeries(timeSeries) {
        const cutoff = Date.now() - timeSeries.retention;
        timeSeries.values = timeSeries.values.filter((value) => value.timestamp > cutoff);
    }
    calculatePercentile(sortedValues, percentile) {
        if (sortedValues.length === 0)
            return 0;
        const index = percentile * (sortedValues.length - 1);
        const lower = Math.floor(index);
        const upper = Math.ceil(index);
        if (lower === upper) {
            return sortedValues[lower];
        }
        const weight = index - lower;
        return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight;
    }
    getDefaultSectionMetrics() {
        return {
            executionCount: 0,
            totalDuration: 0,
            avgDuration: 0,
            maxDuration: 0,
            errorCount: 0,
            successRate: 100,
            lastExecution: 0,
        };
    }
    initializeDefaultMetrics() {
        // Counter metrics
        this.registerMetric({
            name: 'requests_total',
            type: MetricType.COUNTER,
            description: 'Total number of requests processed',
            unit: 'requests',
            labels: ['success'],
        });
        this.registerMetric({
            name: 'rows_processed_total',
            type: MetricType.COUNTER,
            description: 'Total number of data rows processed',
            unit: 'rows',
        });
        this.registerMetric({
            name: 'bytes_processed_total',
            type: MetricType.COUNTER,
            description: 'Total number of bytes processed',
            unit: 'bytes',
        });
        this.registerMetric({
            name: 'files_processed_total',
            type: MetricType.COUNTER,
            description: 'Total number of files processed',
            unit: 'files',
        });
        this.registerMetric({
            name: 'section_executions_total',
            type: MetricType.COUNTER,
            description: 'Total number of section executions',
            unit: 'executions',
            labels: ['section', 'success'],
        });
        // Gauge metrics
        this.registerMetric({
            name: 'memory_heap_used_bytes',
            type: MetricType.GAUGE,
            description: 'Current heap memory usage in bytes',
            unit: 'bytes',
        });
        this.registerMetric({
            name: 'memory_heap_total_bytes',
            type: MetricType.GAUGE,
            description: 'Total heap memory available in bytes',
            unit: 'bytes',
        });
        this.registerMetric({
            name: 'memory_external_bytes',
            type: MetricType.GAUGE,
            description: 'External memory usage in bytes',
            unit: 'bytes',
        });
        this.registerMetric({
            name: 'memory_rss_bytes',
            type: MetricType.GAUGE,
            description: 'Resident set size in bytes',
            unit: 'bytes',
        });
        this.registerMetric({
            name: 'cpu_usage_seconds',
            type: MetricType.GAUGE,
            description: 'CPU usage in seconds',
            unit: 'seconds',
        });
        this.registerMetric({
            name: 'processing_rate_rows_per_second',
            type: MetricType.GAUGE,
            description: 'Current data processing rate in rows per second',
            unit: 'rows/second',
        });
        this.registerMetric({
            name: 'throughput_operations_per_second',
            type: MetricType.GAUGE,
            description: 'Current operation throughput per second',
            unit: 'operations/second',
        });
        this.registerMetric({
            name: 'error_rate_percent',
            type: MetricType.GAUGE,
            description: 'Current error rate as percentage',
            unit: 'percent',
        });
        this.registerMetric({
            name: 'errors_total',
            type: MetricType.GAUGE,
            description: 'Total number of errors encountered',
            unit: 'errors',
        });
        this.registerMetric({
            name: 'critical_errors_total',
            type: MetricType.GAUGE,
            description: 'Total number of critical errors',
            unit: 'errors',
        });
        this.registerMetric({
            name: 'recovered_errors_total',
            type: MetricType.GAUGE,
            description: 'Total number of recovered errors',
            unit: 'errors',
        });
        // Histogram metrics
        this.registerMetric({
            name: 'request_duration_ms',
            type: MetricType.HISTOGRAM,
            description: 'Request duration in milliseconds',
            unit: 'milliseconds',
            buckets: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000],
        });
        this.registerMetric({
            name: 'section_duration_ms',
            type: MetricType.HISTOGRAM,
            description: 'Section execution duration in milliseconds',
            unit: 'milliseconds',
            labels: ['section'],
            buckets: [10, 50, 100, 500, 1000, 5000, 10000, 30000, 60000],
        });
        logger_1.logger.info('Default metrics registered', { operation: 'initializeDefaultMetrics' });
    }
    initializeSectionMetrics() {
        const sections = ['section1', 'section2', 'section3', 'section4', 'section5', 'section6'];
        sections.forEach((section) => {
            this.sectionMetrics.set(section, this.getDefaultSectionMetrics());
        });
    }
}
exports.MetricsCollector = MetricsCollector;
/**
 * Global metrics collector instance
 */
exports.globalMetricsCollector = MetricsCollector.getInstance();
/**
 * Metrics collection utilities
 */
class MetricsUtils {
    /**
     * Create a timing decorator for automatic duration tracking
     */
    static timing(metricName, labels) {
        return function (target, propertyName, descriptor) {
            const method = descriptor.value;
            descriptor.value = async function (...args) {
                const endTimer = exports.globalMetricsCollector.startTimer(metricName);
                try {
                    const result = await method.apply(this, args);
                    endTimer();
                    return result;
                }
                catch (error) {
                    endTimer();
                    throw error;
                }
            };
        };
    }
    /**
     * Create a counter decorator for automatic counting
     */
    static counter(metricName, labels) {
        return function (target, propertyName, descriptor) {
            const method = descriptor.value;
            descriptor.value = async function (...args) {
                try {
                    const result = await method.apply(this, args);
                    exports.globalMetricsCollector.incrementCounter(metricName, 1, { ...labels, success: 'true' });
                    return result;
                }
                catch (error) {
                    exports.globalMetricsCollector.incrementCounter(metricName, 1, { ...labels, success: 'false' });
                    throw error;
                }
            };
        };
    }
    /**
     * Create metrics for data processing operations
     */
    static recordDataOperation(operationType, rowCount, byteCount, duration, success) {
        exports.globalMetricsCollector.recordDataProcessing(rowCount, byteCount, operationType === 'file');
        exports.globalMetricsCollector.recordHistogram('data_operation_duration_ms', duration, {
            operation: operationType,
            success: success.toString(),
        });
        exports.globalMetricsCollector.incrementCounter('data_operations_total', 1, {
            operation: operationType,
            success: success.toString(),
        });
    }
}
exports.MetricsUtils = MetricsUtils;
//# sourceMappingURL=metrics-collector.js.map