"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.generateDatasetFromDocuments = void 0;
/* eslint-disable @typescript-eslint/no-unused-vars */
const child_process_1 = require("child_process");
const decimal_js_1 = __importDefault(require("decimal.js"));
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const get_upgrade_meta_1 = require("../../../util/get-upgrade-meta");
const restriction_message_1 = require("../../constants/restriction.message");
const callControlPlaneApi_1 = require("../../../util/callContolPlane/callControlPlaneApi");
const generateDatasetFromDocuments = async (req, res) => {
    try {
        // Check if this is a multipart form data request
        const contentType = req.headers['content-type'] || '';
        if (!contentType.includes('multipart/form-data')) {
            res.status(400).json({ error: 'Request must be multipart/form-data' });
            return;
        }
        // Create temp directory for uploads
        const tempDir = path_1.default.join(__dirname, '../../../temp_uploads');
        if (!fs_1.default.existsSync(tempDir)) {
            fs_1.default.mkdirSync(tempDir, { recursive: true });
        }
        // Parse the multipart form data manually
        const chunks = [];
        req.on('data', (chunk) => {
            chunks.push(chunk);
        });
        req.on('end', async () => {
            try {
                const buffer = Buffer.concat(chunks);
                const boundary = contentType.split('boundary=')[1];
                if (!boundary) {
                    res.status(400).json({ error: 'No boundary found in multipart data' });
                    return;
                }
                // Parse multipart data and extract files
                const boundaryBuffer = Buffer.from(`--${boundary}`, 'utf8');
                const endBoundaryBuffer = Buffer.from(`--${boundary}--`, 'utf8');
                const documentPaths = [];
                let includeExpectedOutput = true;
                let numSamples = 5;
                let projectId = '';
                let orgId = '';
                // Find all boundary positions
                let startPos = 0;
                while (true) {
                    const boundaryPos = buffer.indexOf(boundaryBuffer, startPos);
                    if (boundaryPos === -1) {
                        break;
                    }
                    const nextBoundaryPos = buffer.indexOf(boundaryBuffer, boundaryPos + boundaryBuffer.length);
                    if (nextBoundaryPos === -1) {
                        break;
                    }
                    // Extract part between boundaries
                    const partBuffer = buffer.slice(boundaryPos + boundaryBuffer.length, nextBoundaryPos);
                    // Find the header section (before the first double CRLF)
                    const doubleCRLF = Buffer.from('\r\n\r\n');
                    const headerEndPos = partBuffer.indexOf(doubleCRLF);
                    if (headerEndPos !== -1) {
                        const headerBuffer = partBuffer.slice(0, headerEndPos);
                        const contentBuffer = partBuffer.slice(headerEndPos + doubleCRLF.length);
                        const header = headerBuffer.toString('utf8');
                        // Parse header to get name and filename
                        const nameMatch = header.match(/name="([^"]+)"/);
                        const filenameMatch = header.match(/filename="([^"]+)"/);
                        if (nameMatch) {
                            const name = nameMatch[1];
                            if (filenameMatch) {
                                // This is a file
                                const filename = filenameMatch[1];
                                if (filename && contentBuffer.length > 0) {
                                    const tempFilePath = path_1.default.join(tempDir, filename);
                                    fs_1.default.writeFileSync(tempFilePath, contentBuffer);
                                    documentPaths.push(tempFilePath);
                                    console.log('Saved file:', filename, 'at:', tempFilePath);
                                }
                            }
                            else {
                                // This is a form field
                                const content = contentBuffer.toString('utf8').replace(/\r\n$/, '');
                                console.log('Form field:', name, '=', content);
                                switch (name) {
                                    case 'includeExpectedOutput':
                                        includeExpectedOutput = content === 'true';
                                        break;
                                    case 'numSamples':
                                        numSamples = Number.parseInt(content) || 5;
                                        break;
                                    case 'projectId':
                                        projectId = content;
                                        break;
                                    case 'orgId':
                                        orgId = content;
                                        break;
                                }
                            }
                        }
                    }
                    startPos = nextBoundaryPos;
                }
                console.log('includeExpectedOutput', includeExpectedOutput);
                console.log('numSamples', numSamples);
                console.log('projectId', projectId);
                console.log('orgId', orgId);
                console.log('documentPaths', documentPaths);
                if (documentPaths.length === 0) {
                    return res.status(400).json({ error: 'At least one document is required' });
                }
                // Validate that all files exist
                for (const docPath of documentPaths) {
                    if (!fs_1.default.existsSync(docPath)) {
                        return res.status(400).json({ error: `Document not found: ${docPath}` });
                    }
                }
                const { restrictionsData, isEnterprise, wallet, notifications, thresholds } = await (0, get_upgrade_meta_1.getUpgradeMetaByOrgId)(orgId);
                if (!isEnterprise &&
                    restrictionsData?.datasets > 0) {
                    return res.status(400).json({
                        success: false,
                        message: restriction_message_1.ERROR_RESTRICTION_MESSAGE.upgrade.title,
                        showToast: true,
                        toastMetadata: {
                            text: restriction_message_1.ERROR_RESTRICTION_MESSAGE.upgrade.message,
                            type: 'error',
                        },
                    });
                }
                if (!isEnterprise &&
                    new decimal_js_1.default(wallet?.credits ?? 0).lte(0)) {
                    if (notifications?.canSendWalletUsageExceeded) {
                        await (0, callControlPlaneApi_1.callControlPlane)(`/trusys-emails/limit-reached`, {
                            method: 'POST',
                            params: {
                                orgId: orgId,
                                type: 'Wallet',
                                currentUsage: wallet?.credits ?? 0,
                            },
                        }, '');
                    }
                    return res.status(400).json({
                        success: false,
                        message: restriction_message_1.ERROR_RESTRICTION_MESSAGE.lowCredits.title,
                        showToast: true,
                        toastMetadata: {
                            text: restriction_message_1.ERROR_RESTRICTION_MESSAGE.lowCredits.message,
                        },
                    });
                }
                if (!isEnterprise &&
                    new decimal_js_1.default(wallet?.credits ?? 0).lte(thresholds?.walletNotificationThreshold ?? 0)
                    && notifications?.canSendLowWalletBalance
                    && notifications?.canSendLowWalletBalance) {
                    await (0, callControlPlaneApi_1.callControlPlane)(`/trusys-emails/limit-reaching`, {
                        method: 'POST',
                        params: {
                            orgId: orgId,
                            type: 'Wallet',
                            currentUsage: wallet?.credits ?? 0,
                        },
                    }, '');
                }
                // Call the Python script with parameters
                const pythonScriptPath = path_1.default.join(__dirname, '../../../python/generate_dataset_from_docs.py');
                // Prepare arguments for Python script
                const documentPathsJson = JSON.stringify(documentPaths);
                const includeExpectedOutputStr = includeExpectedOutput ? 'true' : 'false';
                const maxGoldensTotal = Math.floor(numSamples); // Total desired samples
                // Calculate per-context value
                // DeepEval can split documents into multiple contexts, so we generate conservatively
                // Python will strictly enforce maxGoldensTotal regardless of how much we generate
                const maxGoldensPerContext = Math.max(2, Math.ceil(maxGoldensTotal / documentPaths.length));
                const maxGoldensPerContextStr = maxGoldensPerContext.toString();
                const maxGoldensTotalStr = maxGoldensTotal.toString();
                console.log(`Dataset generation params: ${maxGoldensTotal} total samples, ${maxGoldensPerContext} per context, ${documentPaths.length} documents`);
                // Detect platform for Python executable
                const isWindows = process.platform === 'win32';
                // Helper to find Python in PATH on Windows
                const findPythonInPath = (command) => {
                    if (!isWindows) {
                        return null;
                    }
                    const pathEnv = process.env.PATH || '';
                    const paths = pathEnv.split(path_1.default.delimiter);
                    for (const dir of paths) {
                        const pythonPath = path_1.default.join(dir, `${command}.exe`);
                        if (fs_1.default.existsSync(pythonPath)) {
                            return pythonPath;
                        }
                    }
                    return null;
                };
                // Try to find the correct Python executable
                const possiblePythonPaths = isWindows
                    ? ['./venv/Scripts/python.exe', './venv/Scripts/python']
                    : ['./venv/bin/python3', './venv/bin/python'];
                let pythonExecutable = null;
                // First, try venv paths
                for (const pythonPath of possiblePythonPaths) {
                    const fullPath = path_1.default.resolve(__dirname, '../../../', pythonPath);
                    if (fs_1.default.existsSync(fullPath)) {
                        pythonExecutable = fullPath;
                        console.log('Found Python executable in venv:', pythonExecutable);
                        break;
                    }
                }
                // If not found in venv, try to find in system PATH
                if (!pythonExecutable) {
                    if (isWindows) {
                        pythonExecutable =
                            findPythonInPath('python') || findPythonInPath('python3') || findPythonInPath('py');
                        if (pythonExecutable) {
                            console.log('Found Python executable in PATH:', pythonExecutable);
                        }
                    }
                    else {
                        // For Unix-like systems (Docker), try common locations
                        // Priority: python3.10 (installed in Dockerfile) -> python3 -> python
                        const unixPaths = [
                            '/usr/local/bin/python3.10',
                            '/usr/bin/python3.10',
                            '/usr/local/bin/python3',
                            '/usr/bin/python3',
                            '/usr/bin/python',
                        ];
                        for (const unixPath of unixPaths) {
                            if (fs_1.default.existsSync(unixPath)) {
                                pythonExecutable = unixPath;
                                console.log('Found Python executable at:', pythonExecutable);
                                break;
                            }
                        }
                    }
                }
                // Fallback to system command (will require shell mode)
                if (!pythonExecutable) {
                    // In Docker/Unix, prefer python3.10, then python3, then python
                    pythonExecutable = isWindows ? 'python' : 'python3.10';
                    console.log('Using fallback Python command:', pythonExecutable);
                }
                console.log('Using Python executable:', pythonExecutable);
                console.log('Python script path:', pythonScriptPath);
                // Verify Python script exists
                if (!fs_1.default.existsSync(pythonScriptPath)) {
                    return res.status(500).json({
                        error: 'Python script not found',
                        details: `Script path: ${pythonScriptPath}`,
                    });
                }
                // Execute the Python script using the found Python executable
                // Use shell mode only if we don't have an absolute path to Python
                const needsShell = !path_1.default.isAbsolute(pythonExecutable);
                const args = [
                    pythonScriptPath,
                    documentPathsJson,
                    includeExpectedOutputStr,
                    maxGoldensPerContextStr,
                    maxGoldensTotalStr, // Pass the total limit
                    '', // Optional previous-rows JSON path (evaluation-time novelty); empty for UI upload flow
                ];
                console.log('Spawning Python with shell:', needsShell);
                console.log('Python args length:', args.length);
                const pythonProcess = (0, child_process_1.spawn)(pythonExecutable, args, {
                    env: {
                        ...process.env, // Pass all current environment variables
                        PYTHONPATH: process.env.PYTHONPATH || '',
                        PATH: process.env.PATH || '',
                    },
                    shell: needsShell,
                    windowsHide: true, // Hide the console window on Windows
                });
                pythonProcess.on('error', (error) => {
                    console.error('Failed to start Python process:', error);
                    // Clean up uploaded files
                    try {
                        for (const docPath of documentPaths) {
                            if (fs_1.default.existsSync(docPath)) {
                                fs_1.default.unlinkSync(docPath);
                            }
                        }
                    }
                    catch (cleanupError) {
                        console.error('Error cleaning up uploaded files:', cleanupError);
                    }
                    return res.status(500).json({
                        error: 'Failed to start Python process',
                        details: error.message,
                        suggestion: 'Please ensure Python is installed and available in your system PATH. On Windows, try running "python --version" or "py --version" in your terminal.',
                    });
                });
                let outputData = '';
                let errorData = '';
                const datasetTokenUsage = {
                    cost: new decimal_js_1.default(0),
                    tokenUsage: {
                        prompt: new decimal_js_1.default(0),
                        completion: new decimal_js_1.default(0),
                        total: new decimal_js_1.default(0),
                        cached: new decimal_js_1.default(0),
                    },
                };
                // Regex to extract API cost: matches "API cost:" followed by decimal number
                const costRegex = /API cost:\s*(\d+\.\d+)/i;
                pythonProcess.stdout.on('data', (data) => {
                    const strData = data.toString();
                    // Extract cost from output
                    const costMatch = strData.match(costRegex);
                    if (costMatch && costMatch[1]) {
                        datasetTokenUsage.cost = new decimal_js_1.default(costMatch[1]);
                    }
                    outputData += strData;
                });
                pythonProcess.stderr.on('data', (data) => {
                    const strData = data.toString();
                    // Extract cost from stderr as well
                    const costMatch = strData.match(costRegex);
                    if (costMatch && costMatch[1]) {
                        datasetTokenUsage.cost = new decimal_js_1.default(costMatch[1]);
                    }
                    errorData += strData;
                    console.log('Python stderr:', strData);
                });
                pythonProcess.on('close', (code) => {
                    console.log('Python process exited with code:', code);
                    console.log('Python stdout:', outputData);
                    console.log('Python stderr:', errorData);
                    // Extract cost from final accumulated output (fallback in case it wasn't caught in data handlers)
                    const allOutput = outputData + errorData;
                    const costMatch = allOutput.match(costRegex);
                    if (costMatch && costMatch[1]) {
                        datasetTokenUsage.cost = new decimal_js_1.default(costMatch[1]);
                    }
                    // Clean up uploaded files
                    try {
                        for (const docPath of documentPaths) {
                            if (fs_1.default.existsSync(docPath)) {
                                fs_1.default.unlinkSync(docPath);
                            }
                        }
                    }
                    catch (error) {
                        console.error('Error cleaning up uploaded files:', error);
                    }
                    if (code !== 0) {
                        console.error('Python script error:', errorData);
                        return res.status(500).json({
                            error: 'Failed to generate dataset from documents',
                            details: errorData,
                        });
                    }
                    let lastJson = null;
                    try {
                        const matches = [...outputData.matchAll(/\{[\s\S]*\}$/gm)];
                        if (matches.length > 0) {
                            lastJson = matches[matches.length - 1][0];
                        }
                        if (!lastJson) {
                            throw new Error('No valid JSON found in Python output');
                        }
                        const result = JSON.parse(lastJson);
                        if (!result.success) {
                            return res.status(500).json({
                                error: 'Dataset generation failed',
                                details: result.error,
                            });
                        }
                        console.log('result', result);
                        // Helper function to clean null bytes and control characters from strings
                        const cleanString = (str) => {
                            if (typeof str === 'string') {
                                // Remove null bytes and other problematic control characters
                                return str.replace(/\u0000/g, '').replace(/[\x00-\x08\x0B-\x0C\x0E-\x1F]/g, '');
                            }
                            if (Array.isArray(str)) {
                                return str.map(cleanString);
                            }
                            if (str && typeof str === 'object') {
                                const cleaned = {};
                                for (const [key, value] of Object.entries(str)) {
                                    cleaned[key] = cleanString(value);
                                }
                                return cleaned;
                            }
                            return str;
                        };
                        // Transform the data to match the expected format
                        const transformedData = {
                            testCaseVars: result.data.map((item, index) => {
                                const cleanedItem = cleanString({
                                    prompt: item.prompt || `Generated prompt ${index + 1}`,
                                    expected_output: item.expected_output || `Generated output ${index + 1}`,
                                    ...item,
                                });
                                return cleanedItem;
                            }),
                            totalRows: result.total_rows,
                            personas: ['Document-based generation'],
                            metadata: cleanString({
                                documentPaths: documentPaths.map((p) => path_1.default.basename(p)),
                                includeExpectedOutput,
                                numSamples,
                                generationMethod: 'documents',
                            }),
                            tokenUsages: datasetTokenUsage,
                        };
                        res.status(200).json(transformedData);
                    }
                    catch (parseError) {
                        console.error('Error parsing Python output:', parseError);
                        res.status(500).json({
                            error: 'Failed to parse generation results',
                            details: outputData,
                        });
                    }
                });
            }
            catch (error) {
                console.error('Error processing multipart data:', error);
                res.status(500).json({ error: 'Failed to process uploaded files' });
            }
        });
    }
    catch (error) {
        console.error('Error generating dataset from documents:', error);
        res.status(500).json({ error: 'Failed to generate dataset from documents' });
    }
};
exports.generateDatasetFromDocuments = generateDatasetFromDocuments;
//# sourceMappingURL=generate-dataset-from-docs.js.map