"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.testRunService = exports.TestRunService = exports.transformAssertion = void 0;
const decimal_js_1 = __importDefault(require("decimal.js"));
const drizzle_orm_1 = require("drizzle-orm");
const lodash_1 = __importDefault(require("lodash"));
const uuid_1 = require("uuid");
const constants_1 = require("../../constants");
const updatedConversation_1 = require("../../conversationSimulator/updatedConversation");
const database_1 = require("../../database");
const tables_1 = require("../../database/tables");
const index_1 = __importDefault(require("../../index"));
const logger_1 = __importDefault(require("../../logger"));
const providers_1 = require("../../providers");
const defaults_1 = require("../../providers/defaults");
const billing_logger_1 = __importDefault(require("./billing/billing-logger"));
const evalJobsCache_1 = require("../../util/evalJobsCache");
const get_upgrade_meta_1 = require("../../util/get-upgrade-meta");
const time_1 = require("../../util/time");
const callControlPlaneApi_1 = require("../../util/callContolPlane/callControlPlaneApi");
const constants_2 = require("../../constants");
const datasetService_1 = require("./datasetService");
const transformDatasetService_1 = require("./transformDatasetService");
const activityLogService_1 = require("./activityLogService");
const generateTextConversationBuilderTests_1 = require("../../util/generateTextConversationBuilderTests");
const generateCallConversationBuilderTests_1 = require("../../util/generateCallConversationBuilderTests");
const restriction_message_1 = require("../constants/restriction.message");
const evalBilling_1 = require("./evaluations/evalBilling");
const attachTrusysAccessTokenToProviders_1 = require("../../util/customAIApplication/attachTrusysAccessTokenToProviders");
const testRunSession_1 = require("../../util/azureSessions/testRunSession");
// Function to format generated prompt content
const formatGeneratedPrompt = (content) => {
    if (typeof content !== 'string') {
        return content;
    }
    return (content
        // Replace literal \n with actual newlines
        .replace(/\\n/g, '\n')
        // Replace literal \t with actual tabs
        .replace(/\\t/g, '\t')
        // Replace literal \r with actual carriage returns
        .replace(/\\r/g, '\r')
        // Clean up markdown formatting
        .replace(/\*\*/g, '') // Remove bold markdown
        .replace(/\*/g, '') // Remove italic markdown
        .replace(/`/g, '') // Remove code markdown
        .replace(/#{1,6}\s/g, '') // Remove heading markdown
        // Clean up extra whitespace
        .replace(/\n\s*\n\s*\n/g, '\n\n') // Replace multiple newlines with double newlines
        .replace(/^\s+|\s+$/g, '') // Trim leading/trailing whitespace
        // Clean up numbered lists that might be malformed
        .replace(/\n\s*\d+\s*\n/g, '\n') // Remove standalone numbers
        .replace(/\n\s*\d+\.\s*/g, '\n• ') // Convert numbered lists to bullet points
        // Clean up any remaining escape sequences
        .replace(/\\(.)/g, '$1')); // Remove backslashes before any character
};
const transformAssertion = (assertion) => {
    if (assertion?.type === 'is-sql') {
        return {
            type: assertion?.type,
            value: {
                databaseType: assertion?.databaseType,
                allowedTables: assertion?.allowedTables?.map((table) => table?.value),
                allowedColumns: assertion?.allowedColumns?.map((column) => column?.value),
            },
        };
    }
    else {
        return assertion;
    }
};
exports.transformAssertion = transformAssertion;
class TestRunService {
    constructor() {
        this.transformDatasetService = new transformDatasetService_1.TransformDatasetService();
    }
    async setTestRunFailed(testRunId) {
        const db = await (0, database_1.getDb)();
        await db.update(tables_1.testRunsTable).set({ status: 'failed' }).where((0, drizzle_orm_1.eq)(tables_1.testRunsTable.id, testRunId));
    }
    /**
     * Create a test run with all validation, processing, and evaluation logic
     */
    async createTestRun(params) {
        try {
            const { evaluateOptions, applicationIds, libraryId, testRunId, apiKey, isCustomProvider = false, orgId, projectId, testSuite, token, userId, userName, } = params;
            const db = await (0, database_1.getDb)();
            const { restrictionsData, isEnterprise, wallet, isStarter, usages, notifications, thresholds, bucketInfo } = await (0, get_upgrade_meta_1.getUpgradeMetaByOrgId)(orgId, {
                fetchMetricUsage: true,
            });
            // block run for reaching the functional evals limit
            if ((isStarter || !isEnterprise) && restrictionsData?.functional_evals > 0) {
                const whereConditions = (0, time_1.createTimespanWhereConditions)({
                    table: tables_1.testRunsTable,
                    timeColumn: 'createdAt',
                    orgId,
                    timespan: restrictionsData?.functional_evals_timespan || 'lifetime',
                    extraConditions: (0, drizzle_orm_1.and)((0, drizzle_orm_1.sql) `${tables_1.testRunsTable.type} = ${constants_1.TEST_RUN_TYPE.EVAL}`, (0, drizzle_orm_1.sql) `${tables_1.testRunsTable.status} = ${constants_1.TEST_RUN_STATUS.COMPLETED}`),
                });
                const allCount = await db
                    .select({ count: (0, drizzle_orm_1.sql) `count(*)` })
                    .from(tables_1.testRunsTable)
                    .where(whereConditions)
                    .then((result) => Number(result[0]?.count ?? 0));
                if (isStarter
                    && new decimal_js_1.default(allCount).gt(0)
                    && new decimal_js_1.default(allCount).gte(restrictionsData?.functional_evals ?? 0)) {
                    if (testRunId) {
                        await this.setTestRunFailed(testRunId);
                    }
                    return {
                        success: false,
                        message: restriction_message_1.ERROR_RESTRICTION_MESSAGE.upgrade.title,
                        showToast: true,
                        toastMetadata: {
                            text: restriction_message_1.ERROR_RESTRICTION_MESSAGE.upgrade.message,
                            type: 'error',
                            action: {
                                label: 'Upgrade',
                                actionType: 'upgrade',
                            },
                            duration: 5000,
                        },
                    };
                }
            }
            // Check bucket charges and calculate extra buckets needed
            if (!isEnterprise
                && !isStarter
                && bucketInfo
                && bucketInfo.metric) {
                const metricBucketSize = bucketInfo.metric.bucketSize ?? 0;
                const metricBucketsCharged = bucketInfo.metric.bucketsCharged ?? 0;
                const metricFreeTierLimit = bucketInfo.metric.freeTierLimit ?? 0;
                const metricCurrentUsage = new decimal_js_1.default(usages?.metric ?? 0).toNumber();
                // Calculate if extra buckets are needed based on current usage
                if (metricBucketSize > 0 && metricCurrentUsage > metricFreeTierLimit) {
                    const usageBeyondFree = new decimal_js_1.default(metricCurrentUsage).minus(metricFreeTierLimit);
                    const alreadyCoveredUsage = new decimal_js_1.default(metricBucketsCharged).mul(metricBucketSize);
                    const uncoveredUsage = usageBeyondFree.minus(alreadyCoveredUsage);
                    const extraBucketsNeeded = uncoveredUsage.gt(0)
                        ? uncoveredUsage.div(metricBucketSize).ceil().toNumber()
                        : 0;
                    if (extraBucketsNeeded > 0) {
                        // Call API to charge bucket - calculates cost, checks wallet, deducts, and updates tracking
                        try {
                            const response = await (0, callControlPlaneApi_1.callControlPlane)(`/trusys/billing-plans/charge-bucket`, {
                                method: 'POST',
                                params: {
                                    orgId,
                                    bucketType: 'metrics',
                                    extraBucketsNeeded,
                                    currentUsage: metricCurrentUsage,
                                },
                            }, '');
                            if (response?.error) {
                                if (testRunId) {
                                    await this.setTestRunFailed(testRunId);
                                }
                                return {
                                    success: false,
                                    message: response.error,
                                    showToast: true,
                                    toastMetadata: {
                                        text: response.error,
                                        type: 'error',
                                        duration: 5000,
                                    },
                                };
                            }
                        }
                        catch (error) {
                            // API throws error if insufficient funds or other issues - halt execution
                            const errorMessage = error instanceof Error ? error.message : String(error);
                            if (testRunId) {
                                await this.setTestRunFailed(testRunId);
                            }
                            return {
                                success: false,
                                message: errorMessage.includes('Insufficient')
                                    ? errorMessage
                                    : 'Failed to charge metric bucket. Please try again.',
                                showToast: true,
                                toastMetadata: {
                                    text: errorMessage.includes('Insufficient')
                                        ? errorMessage
                                        : 'Failed to charge metric bucket. Please try again.',
                                    type: 'error',
                                    duration: 5000,
                                },
                            };
                        }
                    }
                }
            }
            // block run for reaching the wallet limit
            if (!isEnterprise
                && new decimal_js_1.default(wallet?.credits ?? 0).lte(new decimal_js_1.default(0))) {
                if (notifications?.canSendWalletUsageExceeded) {
                    await (0, callControlPlaneApi_1.callControlPlane)(`/trusys-emails/limit-reached`, {
                        method: 'POST',
                        params: {
                            orgId,
                            type: 'Wallet',
                            currentUsage: wallet?.credits ?? 0,
                        },
                    }, '');
                }
                if (testRunId) {
                    await this.setTestRunFailed(testRunId);
                }
                return {
                    success: false,
                    message: restriction_message_1.ERROR_RESTRICTION_MESSAGE.lowCredits.title,
                    showToast: true,
                    toastMetadata: {
                        text: restriction_message_1.ERROR_RESTRICTION_MESSAGE.lowCredits.message,
                        type: 'error',
                        duration: 5000,
                    },
                };
            }
            // send notification for low wallet balance
            if (!isEnterprise
                && new decimal_js_1.default(wallet?.credits ?? 0).lte(thresholds?.walletNotificationThreshold ?? 0)
                && notifications?.canSendLowWalletBalance) {
                await (0, callControlPlaneApi_1.callControlPlane)(`/trusys-emails/limit-reaching`, {
                    method: 'POST',
                    params: {
                        orgId,
                        type: 'Wallet',
                        currentUsage: wallet?.credits ?? 0,
                    },
                }, '');
            }
            // check for metric usage limit
            if (!isEnterprise
                && restrictionsData?.metrics > 0
                && new decimal_js_1.default(usages?.metric ?? 0).gte(restrictionsData?.metrics ?? 0)) {
                if (isStarter) {
                    if (notifications?.canSendMetricUsageExceeded) {
                        await (0, callControlPlaneApi_1.callControlPlane)(`/trusys-emails/limit-reached`, {
                            method: 'POST',
                            params: {
                                orgId,
                                type: 'Metric',
                                currentUsage: usages?.metric ?? 0,
                            },
                        }, '');
                    }
                    if (testRunId) {
                        await this.setTestRunFailed(testRunId);
                    }
                    return {
                        success: false,
                        message: restriction_message_1.ERROR_RESTRICTION_MESSAGE.metrics.title,
                        showToast: true,
                        toastMetadata: {
                            text: restriction_message_1.ERROR_RESTRICTION_MESSAGE.metrics.message,
                            type: 'error',
                            duration: 5000,
                        },
                    };
                }
                else if (notifications?.canSendMetricRenewal) {
                    await (0, callControlPlaneApi_1.callControlPlane)(`/trusys-emails/limit-renewal`, {
                        method: 'POST',
                        params: {
                            orgId,
                            type: 'Metric',
                            currentUsage: usages?.metric ?? 0,
                        },
                    }, '');
                }
            }
            // Generate test run ID early (needed for validations and database insert)
            const test_run_Id = testRunId ?? (0, uuid_1.v4)();
            // First, create/update the test run record in database
            if (testRunId) {
                await db
                    .update(tables_1.testRunsTable)
                    .set({ status: 'pending' })
                    .where((0, drizzle_orm_1.eq)(tables_1.testRunsTable.id, test_run_Id));
            }
            else {
                await db.insert(tables_1.testRunsTable).values({
                    id: test_run_Id,
                    status: 'pending',
                    projectId,
                    workspaceId: orgId,
                    applicationIds,
                    libraryId,
                });
            }
            await (0, evalJobsCache_1.setEvalJob)(test_run_Id, {
                evalId: null,
                status: 'in-progress',
                progress: 0,
                total: 0,
                result: null,
                logs: [],
            });
            // Return immediately
            const id = test_run_Id;
            // Run all other functionalities in background
            setImmediate(async () => {
                try {
                    await this.processTestRunInBackground({
                        evaluateOptions,
                        applicationIds,
                        libraryId,
                        testRunId: test_run_Id,
                        apiKey,
                        isCustomProvider,
                        orgId,
                        projectId,
                        testSuite,
                        token,
                        userId,
                        userName,
                        isEnterprise, isStarter, usages, bucketInfo
                    });
                }
                catch (error) {
                    logger_1.default.error(`Error processing test run in background: ${error.message}`);
                    // Update test run status to failed on error
                    try {
                        if (test_run_Id) {
                            await this.setTestRunFailed(test_run_Id);
                        }
                    }
                    catch (updateError) {
                        logger_1.default.error(`Error updating test run status to failed: ${updateError}`);
                    }
                }
            });
            return { id, testRunId: test_run_Id };
        }
        catch (error) {
            logger_1.default.error(`Error while test run: ${error.message}`);
            throw error;
        }
    }
    /**
     * Process test run in background - handles all validation, processing, and evaluation
     */
    async processTestRunInBackground(params) {
        const { evaluateOptions, applicationIds, libraryId, testRunId: test_run_Id, apiKey, isCustomProvider = false, orgId, projectId, testSuite, token, userId, userName, isEnterprise, isStarter, usages, bucketInfo } = params;
        try {
            const db = await (0, database_1.getDb)();
            const { tests, datasetIds } = testSuite;
            let processedTestSuite = testSuite;
            const loggingParams = {
                workspaceId: orgId,
                projectId,
                applicationIds,
                libraryId,
                testRunId: test_run_Id,
                libraryName: '',
            };
            // Generate tests for conversation_builder before dataset transformation
            if (testSuite.promptType === 'conversation_builder') {
                if (isCustomProvider) {
                    // Check if aiConvGenCost is already present in testSuite (e.g., from CLI)
                    // If present, log it before generating conversations
                    if (processedTestSuite.aiConvGenCost && processedTestSuite.aiConvGenCost.tokenUsage?.total > 0) {
                        billing_logger_1.default.logFunctionEvalUsage({
                            workspaceId: loggingParams.workspaceId,
                            functionEvalId: test_run_Id,
                            cost: processedTestSuite.aiConvGenCost.cost || 0,
                            tokenUsage: {
                                prompt: processedTestSuite.aiConvGenCost.tokenUsage.prompt || 0,
                                completion: processedTestSuite.aiConvGenCost.tokenUsage.completion || 0,
                                total: processedTestSuite.aiConvGenCost.tokenUsage.total || 0,
                            },
                            metadata: {
                                subType: 'prompt',
                                applicationIds: loggingParams.applicationIds,
                                testRunId: loggingParams.testRunId,
                                libraryId: loggingParams.libraryId,
                            },
                            model: 'gpt-4.1',
                        });
                    }
                    processedTestSuite.prompts = JSON.parse(processedTestSuite.prompts);
                }
                else {
                    const conversationBuilderType = processedTestSuite.conversation_builder_type || processedTestSuite.conversationBuilderType || 'text'; // Default to 'text' for backward compatibility
                    const isTextConversation = conversationBuilderType === 'text';
                    if (isTextConversation) {
                        const { generatedConversations, aiConvGenCost } = await (0, generateTextConversationBuilderTests_1.generateTextConversationBuilderTests)(processedTestSuite, token, apiKey, loggingParams, undefined);
                        // Log billing if AI generation was used
                        if (aiConvGenCost.tokenUsage.total.gt(0)) {
                            billing_logger_1.default.logFunctionEvalUsage({
                                workspaceId: loggingParams.workspaceId,
                                functionEvalId: test_run_Id,
                                cost: aiConvGenCost.cost.toNumber(),
                                tokenUsage: {
                                    prompt: aiConvGenCost.tokenUsage.prompt.toNumber(),
                                    completion: aiConvGenCost.tokenUsage.completion.toNumber(),
                                    total: aiConvGenCost.tokenUsage.total.toNumber(),
                                },
                                metadata: {
                                    subType: 'prompt',
                                    applicationIds: loggingParams.applicationIds,
                                    testRunId: loggingParams.testRunId,
                                    libraryId: loggingParams.libraryId,
                                },
                                model: 'gpt-4.1',
                            });
                        }
                        // Transform to test cases format (same as conversation_ai)
                        if (generatedConversations.length > 0) {
                            processedTestSuite.prompts = generatedConversations.map((conversation) => conversation.turns);
                            processedTestSuite.isTextConversationPrompt = true;
                            processedTestSuite.description = `${processedTestSuite.description || 'Text Conversation Builder Test'} - ${generatedConversations.length} Test Cases`;
                        }
                    }
                    else {
                        // Call conversation (existing logic)
                        const { generatedTests, aiSpeaksFirst } = await (0, generateCallConversationBuilderTests_1.generateCallConversationBuilderTests)(processedTestSuite, token, loggingParams, apiKey);
                        // Replace tests with generated tests if any were created
                        if (generatedTests.length > 0) {
                            processedTestSuite.prompts = ['{{prompt}}'];
                            processedTestSuite.tests = generatedTests;
                            processedTestSuite.aiSpeaksFirst = aiSpeaksFirst;
                            processedTestSuite.description = `${processedTestSuite.description || 'Conversation Builder Test'} - ${generatedTests.length} Test Cases`;
                        }
                        evaluateOptions.maxConcurrency = 1;
                    }
                }
            }
            processedTestSuite.localProvider = isCustomProvider;
            if (!isCustomProvider) {
                try {
                    const transformOptions = {
                        tests: processedTestSuite.tests || tests,
                        datasetIds,
                        token,
                        apiKey,
                        orgId,
                        projectId,
                        parentPromptId: processedTestSuite?.parentPromptId ?? '',
                        testRunId: test_run_Id,
                        prompts: processedTestSuite.prompts, // Pass prompts from test suite
                    };
                    let flattenedTests = await this.transformDatasetService.transformDataset(transformOptions);
                    flattenedTests = flattenedTests.map((test) => ({
                        ...test,
                        assert: Array.isArray(test?.assert)
                            ? test?.assert?.map((assertion) => (0, exports.transformAssertion)(assertion))
                            : test?.assert,
                        parentPromptId: processedTestSuite?.parentPromptId,
                    }));
                    processedTestSuite = {
                        ...processedTestSuite,
                        tests: flattenedTests,
                    };
                }
                catch (error) {
                    logger_1.default.error(`Error in transformDataset: ${error.message}`);
                    // Mark test run as failed
                    await this.setTestRunFailed(test_run_Id);
                    // If it's a restriction error, provide more context
                    if (error instanceof datasetService_1.DatasetRestrictionError) {
                        throw new Error(`Dataset generation failed due to restriction: ${error.message}. Test run marked as failed.`);
                    }
                    throw error; // Re-throw to stop processing
                }
            }
            const providersWithAuth = (0, attachTrusysAccessTokenToProviders_1.attachTrusysAccessTokenToAzureCustomAIProviders)(processedTestSuite.providers, token, test_run_Id, apiKey);
            processedTestSuite = {
                ...processedTestSuite,
                providers: providersWithAuth,
            };
            const loadedProviders = await (0, providers_1.loadApiProviders)(providersWithAuth, {
                env: processedTestSuite.env,
            });
            if (testSuite.promptType === 'conversation_ai' && !isCustomProvider) {
                const conversationParams = {
                    userIntentions: processedTestSuite.userIntentions || [],
                };
                if (processedTestSuite.userProfiles) {
                    conversationParams.userProfiles = processedTestSuite.userProfiles;
                }
                if (processedTestSuite.openingMessage) {
                    conversationParams.openingMessage = processedTestSuite.openingMessage;
                }
                if (processedTestSuite.minimumTurns) {
                    conversationParams.minimumTurns = processedTestSuite.minimumTurns;
                }
                if (processedTestSuite.maximumTurns) {
                    conversationParams.maximumTurns = processedTestSuite.maximumTurns;
                }
                if (processedTestSuite.chatStopCriteria) {
                    conversationParams.chatStopCriteria = processedTestSuite.chatStopCriteria;
                }
                const callbackModel = {
                    async generate(prompt) {
                        const response = await loadedProviders[0].callApi(prompt);
                        return {
                            result: response.output,
                            cost: response.tokenUsage?.total ?? 0,
                        };
                    },
                };
                const azureProvider = (await (0, defaults_1.getDefaultProviders)(undefined, orgId)).synthesizeProvider;
                const aiConvGenCost = {
                    cost: new decimal_js_1.default(0),
                    tokenUsage: {
                        prompt: new decimal_js_1.default(0),
                        completion: new decimal_js_1.default(0),
                        total: new decimal_js_1.default(0),
                        cached: new decimal_js_1.default(0),
                    },
                };
                const generatedPrompt = await (0, updatedConversation_1.runConversationSimulation)(callbackModel, azureProvider, conversationParams, aiConvGenCost);
                console.log('aiConvGenCost', JSON.stringify(aiConvGenCost, null, 2));
                if (aiConvGenCost.tokenUsage.total.gt(0)) {
                    const libraryData = await db.select().from(tables_1.promptsLibraryTable).where((0, drizzle_orm_1.eq)(tables_1.promptsLibraryTable.id, libraryId));
                    if (libraryData && Array.isArray(libraryData) && libraryData.length > 0) {
                        loggingParams.libraryName = libraryData[0]?.name ?? '';
                    }
                    billing_logger_1.default.logFunctionEvalUsage({
                        workspaceId: loggingParams.workspaceId,
                        functionEvalId: test_run_Id,
                        cost: aiConvGenCost.cost.toNumber(),
                        tokenUsage: {
                            prompt: aiConvGenCost.tokenUsage.prompt.toNumber(),
                            completion: aiConvGenCost.tokenUsage.completion.toNumber(),
                            total: aiConvGenCost.tokenUsage.total.toNumber(),
                        },
                        metadata: {
                            subType: 'prompt',
                            applicationIds: loggingParams.applicationIds,
                            testRunId: loggingParams.testRunId,
                            libraryId: loggingParams.libraryId,
                            libraryName: loggingParams.libraryName,
                        },
                        model: 'gpt-4.1',
                    });
                }
                console.log('generatedPrompt', JSON.stringify(generatedPrompt, null, 2));
                let processedConversations = [];
                console.log('generatedPrompt type', typeof generatedPrompt);
                if (typeof generatedPrompt === 'string') {
                    try {
                        // Format the generated prompt before parsing
                        const formattedPrompt = formatGeneratedPrompt(generatedPrompt);
                        const parsed = JSON.parse(formattedPrompt);
                        if (Array.isArray(parsed)) {
                            processedConversations = parsed;
                        }
                    }
                    catch (parseError) {
                        console.error('Failed to parse conversation response:', parseError);
                        // Format the prompt even if JSON parsing fails
                        const formattedPrompt = formatGeneratedPrompt(generatedPrompt);
                        processedConversations = [
                            {
                                turns: [
                                    { role: 'user', content: formattedPrompt },
                                    {
                                        role: 'assistant',
                                        content: 'I understand your request. Let me help you with that.',
                                    },
                                ],
                            },
                        ];
                    }
                }
                else if (Array.isArray(generatedPrompt)) {
                    processedConversations = generatedPrompt.map((conversation) => {
                        if (conversation.turns && Array.isArray(conversation.turns)) {
                            return {
                                ...conversation,
                                turns: conversation.turns.map((turn) => ({
                                    ...turn,
                                    content: formatGeneratedPrompt(turn.content),
                                })),
                            };
                        }
                        return conversation;
                    });
                }
                if (processedConversations.length > 0) {
                    processedTestSuite = {
                        ...processedTestSuite,
                        prompts: processedConversations.map((conversation) => {
                            if (conversation.turns) {
                                return conversation.turns;
                            }
                            else if (Array.isArray(conversation)) {
                                return conversation;
                            }
                            return conversation;
                        }),
                        description: `${processedTestSuite.description || 'Conversation Test'} - ${processedConversations.length} Conversations`,
                    };
                }
            }
            console.log('updated processedTestSuite', JSON.stringify(processedTestSuite, null, 2));
            let currentMetricUsage = new decimal_js_1.default(usages?.metric ?? 0);
            let currentBucketCharged = new decimal_js_1.default(bucketInfo?.metric?.bucketsCharged ?? 0);
            const cleanupTestRunSessions = async () => {
                await (0, testRunSession_1.cleanupAzureSessionsForTestRun)(providersWithAuth, test_run_Id);
            };
            index_1.default
                .evaluate(Object.assign({}, processedTestSuite, {
                writeLatestResults: true,
                sharing: processedTestSuite.sharing ?? true,
            }), Object.assign({}, evaluateOptions, {
                eventSource: 'web',
                progressCallback: async (progress, total) => {
                    const job = await (0, evalJobsCache_1.getEvalJob)(test_run_Id);
                    if (job) {
                        await (0, evalJobsCache_1.updateEvalJob)(test_run_Id, { progress, total });
                        console.log(`[${test_run_Id}] ${progress}/${total}`);
                    }
                },
                testRunId: test_run_Id,
                workspaceId: orgId,
            }))
                .then(async (result) => {
                const job = await (0, evalJobsCache_1.getEvalJob)(test_run_Id);
                if (job) {
                    const summary = await result.toEvaluateSummary();
                    await (0, evalJobsCache_1.updateEvalJob)(test_run_Id, {
                        status: 'complete',
                        result: summary,
                        evalId: result.id,
                    });
                    // Update test run status and link to eval
                    await db.transaction(async (tx) => {
                        await tx
                            .update(tables_1.testRunsTable)
                            .set({ status: 'completed' })
                            .where((0, drizzle_orm_1.eq)(tables_1.testRunsTable.id, test_run_Id));
                        await tx
                            .insert(tables_1.testRunsToEvalsTable)
                            .values({ testRunId: test_run_Id, evalId: result.id })
                            .onConflictDoNothing()
                            .execute();
                        if (!lodash_1.default.isEmpty(datasetIds)) {
                            // Prepare insert values
                            const insertPairs = datasetIds.map((datasetId) => ({
                                testRunId: test_run_Id,
                                datasetId,
                            }));
                            await tx
                                .insert(tables_1.testRunsToDatasetsTable)
                                .values(insertPairs)
                                .onConflictDoNothing({
                                target: [tables_1.testRunsToDatasetsTable.testRunId, tables_1.testRunsToDatasetsTable.datasetId], // unique constraint columns
                            });
                        }
                    });
                    let metricCount = 0;
                    if (Array.isArray(result?.results) && result?.results?.length > 0) {
                        lodash_1.default.forEach(result?.results, (evalResult) => {
                            if (evalResult?.testCase &&
                                Array.isArray(evalResult?.testCase?.assert) &&
                                evalResult?.testCase?.assert?.length > 0) {
                                metricCount += evalResult?.testCase?.assert?.length ?? 0;
                            }
                        });
                        // Aggregate and log usage per model (including Deepgram duration)
                        const { totalCost } = await (0, evalBilling_1.aggregateAndLogEvalUsage)({
                            evalType: 'functional',
                            workspaceId: orgId,
                            evalId: test_run_Id,
                            evalResults: result.results,
                            baseMetadata: {
                                projectId,
                                applicationIds,
                                libraryId,
                                testRunId: test_run_Id,
                                evalId: result.id,
                            },
                        });
                        // Log metric usage once per run
                        if (metricCount > 0) {
                            billing_logger_1.default.logMetricUsage({
                                workspaceId: orgId,
                                testRunId: test_run_Id,
                                cost: totalCost.toNumber(),
                                tokenUsage: {},
                                amount: metricCount,
                                metadata: {
                                    subType: 'functional_evaluation',
                                    projectId,
                                    applicationIds,
                                    libraryId,
                                    testRunId: test_run_Id,
                                    evalId: result.id,
                                },
                            });
                        }
                    }
                    // Check bucket charges and calculate extra buckets needed
                    if (!isEnterprise
                        && !isStarter
                        && bucketInfo
                        && bucketInfo.metric) {
                        const metricBucketSize = bucketInfo.metric.bucketSize ?? 0;
                        const metricFreeTierLimit = bucketInfo.metric.freeTierLimit ?? 0;
                        const metricCurrentUsage = new decimal_js_1.default(currentMetricUsage).plus(metricCount).toNumber();
                        // Calculate if extra buckets are needed based on current usage
                        if (metricBucketSize > 0 && metricCurrentUsage > metricFreeTierLimit) {
                            const usageBeyondFree = new decimal_js_1.default(metricCurrentUsage).minus(metricFreeTierLimit);
                            const alreadyCoveredUsage = new decimal_js_1.default(currentBucketCharged).mul(metricBucketSize);
                            const uncoveredUsage = usageBeyondFree.minus(alreadyCoveredUsage);
                            const extraBucketsNeeded = uncoveredUsage.gt(0)
                                ? uncoveredUsage.div(metricBucketSize).ceil().toNumber()
                                : 0;
                            if (extraBucketsNeeded > 0) {
                                // Call API to charge bucket - calculates cost, checks wallet, deducts, and updates tracking
                                try {
                                    (0, callControlPlaneApi_1.callControlPlane)(`/trusys/billing-plans/charge-bucket`, {
                                        method: 'POST',
                                        params: {
                                            orgId,
                                            bucketType: 'metrics',
                                            extraBucketsNeeded,
                                            currentUsage: metricCurrentUsage,
                                            allowNegative: true,
                                        },
                                        retryCount: 3,
                                    }, '');
                                }
                                catch (error) {
                                    console.error('Error charging metric bucket:', error);
                                }
                                currentMetricUsage = currentMetricUsage.plus(metricCount);
                                currentBucketCharged = currentBucketCharged.plus(extraBucketsNeeded);
                            }
                        }
                    }
                    console.log(`[${test_run_Id}] Complete`);
                    // Log activity for functional evaluation
                    try {
                        const [orgName, projectName] = await Promise.all([
                            (0, activityLogService_1.fetchOrgName)(token, orgId),
                            (0, activityLogService_1.fetchProjectName)(token, orgId, projectId),
                        ]);
                        // Format test run ID for display (e.g., "Functional Eval#test_run_Id")
                        const testRunDisplayId = test_run_Id;
                        if (token) {
                            await (0, activityLogService_1.logActivity)(token, {
                                type: constants_2.ActivityType.RUN_FUNCTIONAL_EVALUATION,
                                params: (0, activityLogService_1.buildActivityParams)({
                                    test_run_id: { value: testRunDisplayId, _id: test_run_Id },
                                    user_name: { value: userName || '', _id: userId || '' },
                                    org_name: { value: orgName || orgId, _id: orgId },
                                    project_name: { value: projectName || projectId, _id: projectId },
                                    application_name: { value: processedTestSuite.providers?.map((provider) => provider.label).join(', ') ?? '' },
                                }),
                                project_id: projectId,
                                org_id: orgId,
                                created_by: userId || '',
                                updated_by: userId || '',
                            });
                        }
                    }
                    catch (activityError) {
                        logger_1.default.error(`Error logging activity for run functional evaluation: ${activityError}`);
                    }
                }
            })
                .catch(async (error) => {
                console.error('Error running evaluation:', error);
                const job = await (0, evalJobsCache_1.getEvalJob)(test_run_Id);
                if (job) {
                    await (0, evalJobsCache_1.updateEvalJob)(test_run_Id, {
                        status: 'error',
                        logs: [...(job.logs || []), `Error: ${error.message}`],
                    });
                }
                // Update test run status to failed
                await this.setTestRunFailed(test_run_Id);
                throw new Error(`Error running evaluation:${error.message}`);
            })
                .finally(async () => {
                await cleanupTestRunSessions();
            });
        }
        catch (error) {
            logger_1.default.error(`Error processing test run in background: ${error.message}`);
            // Update test run status to failed on error
            await this.setTestRunFailed(params.testRunId);
            throw error;
        }
    }
    /**
     * Generate test suite by transforming dataset
     */
    async generateTestSuite(params) {
        try {
            const { tests, datasetIds, token, apiKey, orgId, projectId, parentPromptId } = params;
            if (!tests || !Array.isArray(tests)) {
                throw new Error('tests is required and must be an array');
            }
            const transformOptions = { tests, datasetIds, token, apiKey, orgId, projectId, parentPromptId };
            const transformDatasetServiceInstance = new transformDatasetService_1.TransformDatasetService();
            const flattenedTests = await transformDatasetServiceInstance.transformDataset(transformOptions);
            // Update assert in flattenedTests with transformAssertion
            const processedTests = flattenedTests?.map((test) => ({
                ...test,
                assert: Array.isArray(test?.assert)
                    ? test?.assert?.map((assertion) => (0, exports.transformAssertion)(assertion))
                    : test?.assert,
                parentPromptId: test?.parentPromptId,
            }));
            return { flattenedTests: processedTests };
        }
        catch (error) {
            console.error('Error in generateTestSuite:', error);
            throw error;
        }
    }
}
exports.TestRunService = TestRunService;
// Export singleton instance
exports.testRunService = new TestRunService();
//# sourceMappingURL=testRunService.js.map