"use strict";
/**
 * Universal Analyzer - Multi-format data analysis orchestrator
 * Integrates with the existing 6-section analysis pipeline
 */
Object.defineProperty(exports, "__esModule", { value: true });
exports.UniversalAnalyzer = void 0;
const parser_registry_1 = require("../parsers/base/parser-registry");
const json_parser_1 = require("../parsers/json-parser");
const csv_parser_adapter_1 = require("../parsers/adapters/csv-parser-adapter");
const excel_parser_1 = require("../parsers/excel-parser");
const tsv_parser_1 = require("../parsers/tsv-parser");
const parquet_parser_1 = require("../parsers/parquet-parser");
const types_1 = require("../core/types");
const logger_1 = require("../utils/logger");
// Import existing analyzers (these remain unchanged)
const overview_1 = require("../analyzers/overview");
const quality_1 = require("../analyzers/quality");
const streaming_analyzer_1 = require("../analyzers/streaming/streaming-analyzer");
const visualization_1 = require("../analyzers/visualization");
const engineering_1 = require("../analyzers/engineering");
const modeling_1 = require("../analyzers/modeling");
/**
 * Universal analyzer that works with any supported data format
 */
class UniversalAnalyzer {
    registry;
    initialized = false;
    constructor() {
        this.registry = parser_registry_1.globalParserRegistry;
    }
    /**
     * Initialize the analyzer with all available parsers
     */
    initializeParsers() {
        if (this.initialized)
            return;
        // Register CSV parser (highest priority for backwards compatibility)
        this.registry.register({
            format: 'csv',
            parserFactory: (options) => (0, csv_parser_adapter_1.createCSVParserAdapter)(options),
            detector: {
                detect: async (filePath) => {
                    const adapter = (0, csv_parser_adapter_1.createCSVParserAdapter)();
                    return adapter.detect(filePath);
                },
                getSupportedExtensions: () => ['.csv'],
                getFormatName: () => 'csv',
            },
            priority: 100,
            extensions: ['.csv'],
        });
        // Register TSV parser
        this.registry.register({
            format: 'tsv',
            parserFactory: (options) => (0, tsv_parser_1.createTSVParser)(options),
            detector: new tsv_parser_1.TSVDetector(),
            priority: 90,
            extensions: ['.tsv', '.tab'],
        });
        // Register JSON parser
        this.registry.register({
            format: 'json',
            parserFactory: (options) => (0, json_parser_1.createJSONParser)(options),
            detector: new json_parser_1.JSONDetector(),
            priority: 80,
            extensions: ['.json', '.jsonl', '.ndjson'],
        });
        // Register Excel parser
        this.registry.register({
            format: 'excel',
            parserFactory: (options) => (0, excel_parser_1.createExcelParser)(options),
            detector: new excel_parser_1.ExcelDetector(),
            priority: 70,
            extensions: ['.xlsx', '.xls', '.xlsm'],
        });
        // Register Parquet parser
        this.registry.register({
            format: 'parquet',
            parserFactory: (options) => (0, parquet_parser_1.createParquetParser)(options),
            detector: new parquet_parser_1.ParquetDetector(),
            priority: 60,
            extensions: ['.parquet'],
        });
        this.initialized = true;
        logger_1.logger.info(`Initialized universal analyzer with ${this.registry.getSupportedFormats().length} formats`);
    }
    /**
     * Analyze any supported file format
     */
    async analyzeFile(filePath, options) {
        this.initializeParsers();
        try {
            // 1. Auto-detect format and get parser
            logger_1.logger.info(`Starting universal analysis for: ${filePath}`);
            // Convert CLIOptions to ParseOptions
            const parseOptions = {
                maxRows: options.maxRows,
                encoding: options.encoding,
                format: options.format,
                delimiter: options.delimiter,
                quote: options.quote,
                hasHeader: options.hasHeader,
                jsonPath: options.jsonPath,
                arrayMode: options.arrayMode,
                flattenObjects: options.flattenObjects,
                sheetName: options.sheetName,
                sheetIndex: options.sheetIndex,
                columns: options.columns,
                rowStart: options.rowStart,
                rowEnd: options.rowEnd,
            };
            const { parser, format, detection } = await this.registry.getParser(filePath, parseOptions);
            logger_1.logger.info(`Detected format: ${format} (confidence: ${(detection.confidence * 100).toFixed(1)}%)`);
            // 2. Validate file can be parsed
            const validation = await parser.validate(filePath);
            if (!validation.canProceed) {
                throw new types_1.DataPilotError(`Cannot parse file: ${validation.errors.join(', ')}`, 'UNIVERSAL_PARSE_ERROR', types_1.ErrorSeverity.HIGH, types_1.ErrorCategory.VALIDATION);
            }
            if (validation.warnings.length > 0) {
                logger_1.logger.warn(`Parsing warnings: ${validation.warnings.join(', ')}`);
            }
            // 3. Convert to common dataset format
            const dataset = await this.parseToDataset(parser, filePath, parseOptions, format, detection);
            // 4. Run the same 6-section analysis pipeline
            const analysisResult = await this.runAnalysisPipeline(dataset, options);
            return {
                success: true,
                exitCode: 0,
                data: analysisResult,
                format: options.format || 'markdown',
                metadata: {
                    command: options.command || 'all',
                    filePath,
                    originalFormat: format,
                    detection: {
                        format,
                        confidence: detection.confidence,
                        metadata: detection.metadata,
                    },
                    parserStats: parser.getStats(),
                    timestamp: new Date().toISOString(),
                    version: '1.2.1', // Multi-format support version
                },
            };
        }
        catch (error) {
            return this.handleAnalysisError(error, filePath);
        }
    }
    /**
     * Parse file using detected parser and convert to universal dataset format
     */
    async parseToDataset(parser, filePath, options, format, detection) {
        const rows = [];
        let headers = [];
        let hasHeader = false;
        logger_1.logger.info('Parsing file to dataset format...');
        // Parse file and collect rows
        for await (const row of parser.parse(filePath, {
            maxRows: options.maxRows,
            hasHeader: options.hasHeader,
            encoding: options.encoding,
            delimiter: options.delimiter,
            quote: options.quote,
            jsonPath: options.jsonPath,
            arrayMode: options.arrayMode,
            flattenObjects: options.flattenObjects,
            sheetName: options.sheetName,
            sheetIndex: options.sheetIndex,
            columns: options.columns,
            rowStart: options.rowStart,
            rowEnd: options.rowEnd,
        })) {
            if (row.index === 0 && !hasHeader) {
                // First row - determine if it's headers or data
                const isHeaderRow = this.detectHeaderRow(row.data, format);
                if (isHeaderRow) {
                    headers = row.data;
                    hasHeader = true;
                    continue;
                }
                else {
                    // Generate column names
                    headers = row.data.map((_, i) => `column_${i + 1}`);
                }
            }
            rows.push(row.data);
        }
        const stats = parser.getStats();
        logger_1.logger.info(`Parsed ${rows.length} rows with ${headers.length} columns`);
        return {
            headers,
            rows,
            metadata: {
                format,
                filePath,
                totalRows: rows.length,
                parserStats: stats,
                detection,
            },
        };
    }
    /**
     * Detect if first row contains headers
     */
    detectHeaderRow(row, format) {
        // Format-specific header detection logic
        switch (format) {
            case 'json':
                // JSON usually has meaningful keys as headers
                return true;
            case 'parquet':
                // Parquet always has schema-defined column names
                return true;
            case 'csv':
                // CSV header detection (existing logic)
                return this.detectCSVHeaders(row);
            default:
                // Default: assume first row is header if it contains non-numeric values
                return row.some((cell) => isNaN(Number(cell)) && cell.trim() !== '');
        }
    }
    detectCSVHeaders(row) {
        // Simple heuristic: if more than half the cells are non-numeric, likely headers
        const nonNumeric = row.filter((cell) => {
            const trimmed = cell.trim();
            return trimmed !== '' && isNaN(Number(trimmed));
        });
        return nonNumeric.length > row.length / 2;
    }
    /**
     * Run the existing 6-section analysis pipeline on the universal dataset
     */
    async runAnalysisPipeline(dataset, options) {
        const results = {};
        // Section 1: Overview Analysis (adapted for universal format)
        if (this.shouldRunSection(1, options)) {
            logger_1.logger.info('Running Section 1: Overview Analysis');
            results.section1 = await this.runSection1Analysis(dataset, options);
        }
        // Section 2: Data Quality Analysis
        if (this.shouldRunSection(2, options)) {
            logger_1.logger.info('Running Section 2: Data Quality Analysis');
            results.section2 = await this.runSection2Analysis(dataset, options);
        }
        // Section 3: Exploratory Data Analysis
        if (this.shouldRunSection(3, options)) {
            logger_1.logger.info('Running Section 3: EDA');
            results.section3 = await this.runSection3Analysis(dataset, options);
        }
        // Section 4: Visualization Intelligence
        if (this.shouldRunSection(4, options)) {
            logger_1.logger.info('Running Section 4: Visualization Intelligence');
            results.section4 = await this.runSection4Analysis(dataset, options);
        }
        // Section 5: Data Engineering
        if (this.shouldRunSection(5, options)) {
            logger_1.logger.info('Running Section 5: Data Engineering');
            results.section5 = await this.runSection5Analysis(dataset, options);
        }
        // Section 6: Modeling Strategy
        if (this.shouldRunSection(6, options)) {
            logger_1.logger.info('Running Section 6: Modeling Strategy');
            results.section6 = await this.runSection6Analysis(dataset, options);
        }
        return results;
    }
    shouldRunSection(sectionNumber, options) {
        if (options.sections) {
            return options.sections.includes(sectionNumber.toString());
        }
        // Default sections based on command
        switch (options.command) {
            case 'overview':
                return sectionNumber === 1;
            case 'quality':
                return sectionNumber === 2;
            case 'eda':
                return sectionNumber === 3;
            case 'visualization':
                return sectionNumber === 4;
            case 'engineering':
                return sectionNumber === 5;
            case 'modeling':
                return sectionNumber === 6;
            case 'all':
                return true;
            default:
                return true;
        }
    }
    // Section analysis methods (these adapt existing analyzers to work with universal dataset)
    async runSection1Analysis(dataset, options) {
        const analyzer = new overview_1.Section1Analyzer({
            enableFileHashing: options.enableHashing !== false,
            includeHostEnvironment: options.includeEnvironment !== false,
            privacyMode: options.privacyMode || 'redacted',
            detailedProfiling: options.verbose || false,
            maxSampleSizeForSparsity: 10000,
        });
        // Section1 expects filePath, command, and analysis sections
        return analyzer.analyze(dataset.metadata.filePath, `datapilot ${options.command || 'analysis'} ${dataset.metadata.filePath}`, []);
    }
    async runSection2Analysis(dataset, options) {
        const analyzer = new quality_1.Section2Analyzer({
            data: dataset.rows,
            headers: dataset.headers,
            columnTypes: dataset.headers.map(() => 'string'),
            rowCount: dataset.rows.length,
            columnCount: dataset.headers.length,
            config: {
                enabledDimensions: ['completeness', 'uniqueness', 'validity'],
                strictMode: false,
                maxOutlierDetection: 100,
                semanticDuplicateThreshold: 0.85,
            },
        });
        return analyzer.analyze();
    }
    async runSection3Analysis(dataset, options) {
        const analyzer = new streaming_analyzer_1.StreamingAnalyzer({
            chunkSize: options.chunkSize || 500,
            memoryThresholdMB: options.memoryLimit || 100,
            maxRowsAnalyzed: options.maxRows || 500000,
            enabledAnalyses: ['univariate', 'bivariate', 'correlations'],
            significanceLevel: 0.05,
            maxCorrelationPairs: 50,
            enableMultivariate: true,
        });
        // Section3 expects a file path, not data stream
        return analyzer.analyzeFile(dataset.metadata.filePath);
    }
    async runSection4Analysis(dataset, options) {
        const analyzer = new visualization_1.Section4Analyzer({
            accessibilityLevel: options.accessibility || 'good',
            complexityThreshold: options.complexity || 'moderate',
            maxRecommendationsPerChart: options.maxRecommendations || 3,
            includeCodeExamples: options.includeCode || false,
            enabledRecommendations: [
                'UNIVARIATE',
                'BIVARIATE',
                'DASHBOARD',
                'ACCESSIBILITY',
                'PERFORMANCE',
            ],
            targetLibraries: ['d3', 'plotly', 'observable'],
        });
        // Section4 needs dependencies from previous sections
        // For now, we'll need to create mock dependencies
        // This will be replaced with actual dependency resolution
        const mockSection1 = {
            overview: { structuralDimensions: { totalDataRows: dataset.rows.length } },
        };
        const mockSection3 = { performanceMetrics: { rowsAnalyzed: dataset.rows.length } };
        return analyzer.analyze(mockSection1, mockSection3);
    }
    async runSection5Analysis(dataset, options) {
        const analyzer = new engineering_1.Section5Analyzer({
            targetDatabaseSystem: options.database || 'postgresql',
            mlFrameworkTarget: options.framework || 'scikit_learn',
        });
        // Section5 needs dependencies from previous sections
        const mockSection1 = {
            overview: { structuralDimensions: { totalDataRows: dataset.rows.length } },
        };
        const mockSection2 = { qualityAudit: { overallScore: 85 } };
        const mockSection3 = { performanceMetrics: { rowsAnalyzed: dataset.rows.length } };
        return analyzer.analyze(mockSection1, mockSection2, mockSection3);
    }
    async runSection6Analysis(dataset, options) {
        const analyzer = new modeling_1.Section6Analyzer({
            focusAreas: options.focus || ['regression', 'binary_classification', 'clustering'],
            complexityPreference: options.complexity || 'moderate',
            interpretabilityRequirement: options.interpretability || 'medium',
        });
        // Section6 needs dependencies from previous sections
        const mockSection1 = {
            overview: { structuralDimensions: { totalDataRows: dataset.rows.length } },
        };
        const mockSection2 = { qualityAudit: { overallScore: 85 } };
        const mockSection3 = { performanceMetrics: { rowsAnalyzed: dataset.rows.length } };
        const mockSection5 = { engineeringGuidance: { recommendations: [] } };
        return analyzer.analyze(mockSection1, mockSection2, mockSection3, mockSection5);
    }
    /**
     * Create async iterable data stream from dataset
     */
    async *createDataStream(dataset) {
        for (const row of dataset.rows) {
            yield row;
        }
    }
    /**
     * Handle analysis errors with helpful suggestions
     */
    handleAnalysisError(error, filePath) {
        const supportedFormats = this.registry.getSupportedFormats();
        const supportedExtensions = this.registry.getSupportedExtensions();
        return {
            success: false,
            exitCode: 1,
            error: error.message,
            suggestions: [
                `Check if file format is supported: ${supportedFormats.join(', ')}`,
                `Supported extensions: ${supportedExtensions.join(', ')}`,
                'Try specifying format explicitly: --format json',
                'Verify file is not corrupted',
                'Check file permissions',
                'Use --help for more information',
            ],
            metadata: {
                filePath,
                supportedFormats,
                supportedExtensions,
                timestamp: new Date().toISOString(),
            },
        };
    }
    /**
     * Get supported formats for help/error messages
     */
    getSupportedFormats() {
        this.initializeParsers();
        return this.registry.getSupportedFormats();
    }
    /**
     * Validate file format is supported
     */
    async validateFile(filePath) {
        this.initializeParsers();
        try {
            const validation = await this.registry.validateFile(filePath);
            if (validation.supported && validation.bestMatch) {
                return {
                    supported: true,
                    format: validation.bestMatch.format,
                    confidence: validation.bestMatch.detection.confidence,
                    suggestions: [],
                };
            }
            else {
                return {
                    supported: false,
                    suggestions: [
                        `File format not supported or confidence too low`,
                        `Supported formats: ${this.getSupportedFormats().join(', ')}`,
                        'Try converting to a supported format',
                        'Check if file is corrupted',
                    ],
                };
            }
        }
        catch (error) {
            return {
                supported: false,
                suggestions: [
                    `Error validating file: ${error.message}`,
                    'Check file exists and is readable',
                    'Verify file format is supported',
                ],
            };
        }
    }
}
exports.UniversalAnalyzer = UniversalAnalyzer;
//# sourceMappingURL=universal-analyzer.js.map