"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.doTrusysEval = doTrusysEval;
exports.trusysEvalCommand = trusysEvalCommand;
const chalk_1 = __importDefault(require("chalk"));
const dedent_1 = __importDefault(require("dedent"));
const lodash_1 = __importDefault(require("lodash"));
const path = __importStar(require("path"));
const zod_1 = require("zod");
const zod_validation_error_1 = require("zod-validation-error");
const cache_1 = require("../cache");
const envars_1 = require("../envars");
const evaluator_1 = require("../evaluator");
const evaluatorHelpers_1 = require("../evaluatorHelpers");
const accounts_1 = require("../globalConfig/accounts");
const cloud_1 = require("../globalConfig/cloud");
const logger_1 = __importDefault(require("../logger"));
const providers_1 = require("../providers");
const types_1 = require("../types");
const types_2 = require("../types");
const util_1 = require("../util");
const invariant_1 = __importDefault(require("../util/invariant"));
const buildProviderForEval_1 = require("../util/customAIApplication/buildProviderForEval");
const customAICliHelpers_1 = require("../util/customAIApplication/customAICliHelpers");
const testSuiteListsConfig_1 = require("../util/testSuiteListsConfig");
const generateTextConversationBuilderTests_1 = require("../util/generateTextConversationBuilderTests");
const callServer_1 = require("./callServer");
const EvalCommandSchema = types_2.CommandLineOptionsSchema.extend({
    help: zod_1.z.boolean().optional(),
    interactiveProviders: zod_1.z.boolean().optional(),
    remote: zod_1.z.boolean().optional(),
    cache: zod_1.z.boolean().optional(),
    noCache: zod_1.z.boolean().optional(),
    local: zod_1.z.boolean().optional(),
}).partial();
async function doTrusysEval(cmdObj) {
    (0, util_1.setupEnv)(cmdObj.envPath);
    const apiKey = cloud_1.cloudConfig.getApiKey() || (0, envars_1.getEnvString)('TRUSYS_API_KEY');
    const appUrl = cloud_1.cloudConfig.getTrusysAppUrl();
    const projectId = (0, accounts_1.getProjectId)();
    const workSpaceId = (0, accounts_1.getWorkSpaceId)();
    if (!projectId || !apiKey) {
        await cloud_1.cloudConfig.delete();
        logger_1.default.info(`Not logged in. Run ${chalk_1.default.bold('trusys auth login')} to login.`);
        process.exitCode = 1;
        return null;
    }
    const apiHost = cloud_1.cloudConfig.getTrusysApiHost();
    //const apiHost = 'http://localhost:15500';
    // Initialize conversation simulator API client for CLI calls
    const conversationSimulatorClient = new callServer_1.ConversationSimulatorApiClient(apiKey, apiHost, projectId);
    let testRunId;
    const promptLibrary = await (0, callServer_1.getPrompLibraryDetils)(cmdObj?.library, apiKey, apiHost, projectId);
    if (lodash_1.default.isEmpty(promptLibrary)) {
        logger_1.default.error(`❌ No Prompt library with name  ${cmdObj?.library}`);
        process.exitCode = 1;
        return null;
    }
    if (promptLibrary?.projectId !== projectId) {
        logger_1.default.error(`❌ Prompt library ${cmdObj?.library} does not belong to project ${projectId}`);
        process.exitCode = 1;
        return null;
    }
    logger_1.default.info(`Prompts Library: ${promptLibrary?.name}`);
    let apps = [];
    const appNames = Array.isArray(cmdObj?.application) ? cmdObj.application : [cmdObj?.application];
    logger_1.default.info(`Applications: ${appNames}`);
    for (const appName of appNames) {
        if (!appName) {
            continue;
        }
        const app = await (0, callServer_1.getApplicationDetils)(appName, apiKey, apiHost, projectId);
        if (app?.projectId !== projectId) {
            logger_1.default.error(`❌ Application ${appName} does not belong to project ${projectId}`);
            process.exitCode = 1;
            return null;
        }
        if (app) {
            apps.push(app);
        }
    }
    if (apps.length === 0) {
        logger_1.default.error('❌ No valid applications found.');
        process.exitCode = 1;
        return null;
    }
    for (const app of apps) {
        if ((0, customAICliHelpers_1.requiresCliApplicationPath)(app) && lodash_1.default.isEmpty(cmdObj?.applicationPath)) {
            logger_1.default.error(`❌ Please provide --applicationPath for shell-mode Custom AI application "${app?.name}". Upload-mode apps run on the server without a local path.`);
            process.exitCode = 1;
            return null;
        }
        if ((0, customAICliHelpers_1.isCustomAIUploadMode)(app) && (cmdObj?.applicationPath || cmdObj?.local)) {
            logger_1.default.warn(`Application "${app?.name}" uses upload-mode Custom AI; --applicationPath and --local are ignored. The server will use the stored bundle.`);
        }
    }
    // Update application path and validate the filepath extention
    if (cmdObj?.applicationPath) {
        const appPaths = Array.isArray(cmdObj?.applicationPath)
            ? cmdObj?.applicationPath
            : [cmdObj?.applicationPath];
        let fileExtensionNotMatch = false;
        let isCustomAIApplicationPresent = false;
        apps = apps?.map((item, index) => {
            const language = item?.provider_config?.config?.provider?.language;
            const providerPath = appPaths[index];
            // logger.info(`Appplication details ${JSON.stringify(item)}`);
            if (item?.provider_config?.config?.custom_provider_type === 'Custom AI Application') {
                if ((0, customAICliHelpers_1.isCustomAIUploadMode)(item)) {
                    return item;
                }
                isCustomAIApplicationPresent = true;
                let hasCorrectExt = false;
                const modulePath = path.isAbsolute(providerPath)
                    ? providerPath
                    : path.join(process.cwd(), providerPath);
                //  logger.info(`modulePath ${modulePath}`);
                const updatedItem = {
                    ...item,
                    provider_config: {
                        ...item?.provider_config,
                        config: {
                            ...item?.provider_config?.config,
                            provider: {
                                ...item?.provider_config?.config?.provider,
                                id: `file://${modulePath}`,
                                type: 'Custom AI Application',
                                config: {
                                    ...item?.provider_config?.config?.provider?.config,
                                    basePath: modulePath,
                                },
                            },
                        },
                    },
                };
                // logger.debug(`updatedItem Appplication details ${JSON.stringify(updatedItem)}`);
                if (language === 'go') {
                    hasCorrectExt = providerPath.endsWith('.go');
                }
                else if (language === 'python') {
                    hasCorrectExt = providerPath.endsWith('.py');
                }
                else if (language === 'js') {
                    hasCorrectExt = providerPath.endsWith('.js') || providerPath.endsWith('.ts') || providerPath.endsWith('.mjs');
                }
                if (!hasCorrectExt) {
                    fileExtensionNotMatch = true;
                    logger_1.default.error(`❌ Please give ${language} file path`);
                    process.exitCode = 1;
                    return null;
                }
                return updatedItem;
            }
            else {
                return item;
            }
        });
        if (isCustomAIApplicationPresent && fileExtensionNotMatch) {
            logger_1.default.error(`❌ Please give a proper file path for custom AI application`);
            process.exitCode = 1;
            return null;
        }
    }
    const testSuiteLists = (0, testSuiteListsConfig_1.generatetestSuiteLists)({
        promptLibrary,
        applications: apps,
        description: '',
        defaultTest: {},
        scenarios: [],
        evaluateOptions: {
            cache: (0, cache_1.isCacheEnabled)(),
        },
    });
    // Run test suite for each prompt
    for (const item of testSuiteLists) {
        const testSuite = {
            ...item,
            ...(testRunId && { testRunId }),
        };
        const providerId = typeof testSuite?.providers[0]?.id === 'function'
            ? testSuite.providers[0].id()
            : (testSuite?.providers[0]?.id ?? '');
        const useLocalCustomProvider = providerId.startsWith('file://') ||
            (Boolean(cmdObj?.local) && providerId !== buildProviderForEval_1.AZURE_SESSIONS_CUSTOM_AI_PROVIDER_ID);
        if (useLocalCustomProvider) {
            testSuite.isCustomProvider = true;
            const providerClass = await (0, providers_1.loadApiProvider)(providerId, {
                env: testSuite?.env,
                basePath: testSuite?.providers[0]?.config?.basePath,
                options: { config: { ...(testSuite?.providers[0]?.config ?? {}) } },
            });
            // Run all test provider calls in parallel and store the response in each test object
            // Remove providerReponse from testSuite, only set on each test object
            if (providerId.startsWith('file://') || cmdObj?.local) {
                // Call getGeneratedTestSuite and update testSuite.tests with the result, then do other logic
                const generatedTestSuite = await (0, callServer_1.getGeneratedTestSuite)(testSuite, apiKey, apiHost, projectId, testSuite?.parentPromptId ?? '');
                if (generatedTestSuite && Array.isArray(generatedTestSuite.flattenedTests)) {
                    testSuite.tests = generatedTestSuite.flattenedTests;
                    // Check if this is a conversation type prompt
                    const isConversationType = testSuite.promptType === 'conversation_builder' &&
                        (testSuite.conversation_builder_type === 'text' || testSuite.conversationBuilderType === 'text');
                    if (isConversationType) {
                        const controlPlaneUrl = cloud_1.cloudConfig.getCpApiHost();
                        const loggingParams = {
                            workspaceId: workSpaceId || '',
                            projectId: projectId || '',
                            applicationIds: apps.map((app) => app.id),
                            libraryId: promptLibrary.id,
                            testRunId: testRunId || '',
                        };
                        const { generatedConversations, aiConvGenCost } = await (0, generateTextConversationBuilderTests_1.generateTextConversationBuilderTests)(testSuite, undefined, apiKey, loggingParams, controlPlaneUrl, conversationSimulatorClient, true);
                        if (generatedConversations.length > 0) {
                            // Use the entire conversation with all turns, same as create-test-run.ts
                            // Map each conversation to its turns (1 conversation with 4 turns = 1 prompt with 4 turns)
                            testSuite.prompts = generatedConversations.map((conversation) => conversation.turns);
                            testSuite.isTextConversationPrompt = true;
                            if (aiConvGenCost) {
                                testSuite.aiConvGenCost = {
                                    cost: aiConvGenCost.cost.toNumber(),
                                    tokenUsage: {
                                        prompt: aiConvGenCost.tokenUsage.prompt.toNumber(),
                                        completion: aiConvGenCost.tokenUsage.completion.toNumber(),
                                        total: aiConvGenCost.tokenUsage.total.toNumber(),
                                        cached: aiConvGenCost.tokenUsage.cached.toNumber(),
                                    },
                                };
                            }
                            // Process each generated conversation as a separate test case
                            // Each conversation (with all its turns) becomes one test case
                            const tests = testSuite?.tests ?? [];
                            // Process each conversation as a test case
                            for (let convIndex = 0; convIndex < generatedConversations.length; convIndex++) {
                                const conversation = generatedConversations[convIndex];
                                const test = convIndex < tests.length ? tests[convIndex] : null;
                                if (conversation?.turns && Array.isArray(conversation.turns) && conversation.turns.length > 0) {
                                    // Get the last assistant response
                                    const lastTurn = conversation.turns[conversation.turns.length - 1];
                                    const lastAssistantResponse = lastTurn?.role === 'assistant' ? lastTurn.content : '';
                                    // Remove last assistant response from conversation
                                    const conversationWithoutLast = conversation.turns.slice(0, -1);
                                    if (test) {
                                        // Set the conversation (without last turn) as the prompt
                                        testSuite.prompts[convIndex] = [...conversationWithoutLast];
                                        // Use the last assistant response as the provider response
                                        test.providerResponse = {
                                            output: lastAssistantResponse,
                                        };
                                        test.isTextConversationPrompt = true;
                                        test.providerOutput = lastAssistantResponse;
                                    }
                                }
                                else if (test) {
                                    // Fallback to original behavior if no conversation
                                    const prompt = {
                                        raw: testSuite?.prompts[0],
                                        label: testSuite?.prompts[0],
                                    };
                                    const renderedPrompt = await (0, evaluatorHelpers_1.renderPrompt)(prompt, test?.vars);
                                    const providerResponse = await providerClass.callApi(renderedPrompt);
                                    test.providerResponse = providerResponse;
                                    test.providerOutput = providerResponse.output;
                                }
                            }
                            testSuite.tests = tests.slice(0, generatedConversations.length);
                            testSuite.prompts = JSON.stringify(testSuite.prompts);
                        }
                        else {
                            // Fallback to original behavior if no conversations generated
                            const tests = testSuite?.tests ?? [];
                            const batchSize = Number(cmdObj?.maxConcurrency) || 20;
                            for (let i = 0; i < tests.length; i += batchSize) {
                                const batch = tests.slice(i, i + batchSize);
                                await Promise.all(batch.map(async (test) => {
                                    const prompt = {
                                        raw: testSuite?.prompts[0],
                                        label: testSuite?.prompts[0],
                                    };
                                    const renderedPrompt = await (0, evaluatorHelpers_1.renderPrompt)(prompt, test?.vars);
                                    const providerResponse = await providerClass.callApi(renderedPrompt);
                                    test.providerResponse = providerResponse;
                                    test.providerOutput = providerResponse.output;
                                    return providerResponse;
                                }));
                            }
                        }
                    }
                    else {
                        // Original behavior for non-conversation prompts
                        // Batch providerClass.callApi calls according to batchsize (cmdObj?.maxConcurrency)
                        {
                            const tests = testSuite?.tests ?? [];
                            const batchSize = Number(cmdObj?.maxConcurrency) || 20;
                            for (let i = 0; i < tests.length; i += batchSize) {
                                const batch = tests.slice(i, i + batchSize);
                                await Promise.all(batch.map(async (test) => {
                                    const prompt = {
                                        raw: testSuite?.prompts[0],
                                        label: testSuite?.prompts[0],
                                    };
                                    const renderedPrompt = await (0, evaluatorHelpers_1.renderPrompt)(prompt, test?.vars);
                                    const providerResponse = await providerClass.callApi(renderedPrompt);
                                    test.providerResponse = providerResponse;
                                    test.providerOutput = providerResponse.output;
                                    return providerResponse;
                                }));
                            }
                        }
                    }
                }
            }
        }
        const result = await (0, callServer_1.runTestSuite)(testSuite, apiKey, apiHost, projectId);
        if (!testRunId && result?.testRunId) {
            testRunId = result?.testRunId;
        }
        if (!result?.success) {
            logger_1.default.error(`Test suite failed: ${result?.error}`);
            process.exitCode = 1;
            return null;
        }
    }
    const shareUrl = `${appUrl}/workspaces/${workSpaceId}/projects/${projectId}/test-runs/${testRunId}`;
    logger_1.default.info(chalk_1.default.green(`✅ Test run completed`));
    logger_1.default.info(chalk_1.default.blue(`Eval test completed: ${shareUrl}`));
    const trusysTestResult = await (0, callServer_1.getTestRunDetails)(testRunId, apiKey, apiHost, projectId);
    const failedTestCount = trusysTestResult?.data?.summary?.stats?.failures;
    if (failedTestCount && failedTestCount > 0) {
        process.exitCode = 100;
    }
    if (cmdObj.output) {
        await (0, util_1.writeTrusysOutput)(cmdObj.output, trusysTestResult?.data, shareUrl);
        logger_1.default.info(chalk_1.default.blue(`Test Run ${testRunId} has been successfully exported to ${cmdObj.output}.`));
        process.exit(0);
    }
    // Exit CLI after test completion
    process.exit(process.exitCode || 0);
}
function trusysEvalCommand(program, defaultConfig, defaultConfigPath) {
    const evaluateOptions = {};
    if (defaultConfig.evaluateOptions) {
        evaluateOptions.generateSuggestions = defaultConfig.evaluateOptions.generateSuggestions;
        evaluateOptions.maxConcurrency = defaultConfig.evaluateOptions.maxConcurrency;
        evaluateOptions.showProgressBar = defaultConfig.evaluateOptions.showProgressBar;
    }
    const evalCmd = program
        .command('eval')
        .description('Evaluate prompts')
        // Execution control
        .option('-j, --max-concurrency <number>', 'Maximum number of concurrent API calls', defaultConfig.evaluateOptions?.maxConcurrency
        ? String(defaultConfig.evaluateOptions.maxConcurrency)
        : `${evaluator_1.DEFAULT_MAX_CONCURRENCY}`)
        .option('--delay <number>', 'Delay between each test (in milliseconds)', defaultConfig.evaluateOptions?.delay ? String(defaultConfig.evaluateOptions.delay) : '0')
        .option('--cache', 'Enable caching for Provider responses')
        .option('--noCache', 'Disable caching for Provider responses')
        // Miscellaneous
        .option('--description <description>', 'Description of the eval run')
        .option('--run', 'Run the eval.')
        .option('--application <application...>', 'Name(s) or Id(s) of the app(s)')
        .option('--library <library>', 'Prompt library to use')
        .option('--applicationPath <paths...>', 'Path(s) to application file(s)')
        .option('--local', 'Run evaluation locally using application file(s)')
        .option('--assertions <path>', 'Path to assertions file')
        .option('--model-outputs <path>', 'Path to file or folder containing LLM outputs')
        .option('-o, --output <outputPath>', 'Output path for the exported file')
        .action(async (opts, command) => {
        let validatedOpts;
        try {
            validatedOpts = EvalCommandSchema.parse(opts);
            // logger.info(`Command Params: ${JSON.stringify(validatedOpts)}`);
        }
        catch (err) {
            const validationError = (0, zod_validation_error_1.fromError)(err);
            logger_1.default.error((0, dedent_1.default) `
        Invalid command options:
        ${validationError.toString()}
        `);
            process.exitCode = 1;
            return;
        }
        if (command.args.length > 0) {
            logger_1.default.warn(`Unknown command: ${command.args[0]}. Did you mean -c ${command.args[0]}?`);
        }
        if (validatedOpts.help) {
            evalCmd.help();
            return;
        }
        if (validatedOpts.output) {
            const { data: extension } = types_1.OutputFileExtension.safeParse(validatedOpts.output.split('.').pop()?.toLowerCase());
            (0, invariant_1.default)(extension, `Unsupported output file format: ${validatedOpts.output}. Please use one of: ${types_1.OutputFileExtension.options.join(', ')}.`);
        }
        if (validatedOpts.run) {
            if (validatedOpts.cache) {
                (0, cache_1.enableCache)();
            }
            else if (validatedOpts.noCache) {
                (0, cache_1.disableCache)();
            }
            else {
                (0, cache_1.disableCache)();
            }
            if (validatedOpts.application && validatedOpts.library) {
                doTrusysEval(validatedOpts);
            }
            else {
                logger_1.default.error('❌ Please provide application name(s) and library name');
            }
            return;
        }
    });
    return evalCmd;
}
//# sourceMappingURL=trusys_eval.js.map