"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DEFAULT_MAX_CONCURRENCY = void 0;
exports.isAllowedPrompt = isAllowedPrompt;
exports.generateVarCombinations = generateVarCombinations;
exports.evaluate = evaluate;
const async_1 = __importDefault(require("async"));
const chalk_1 = __importDefault(require("chalk"));
const crypto_1 = require("crypto");
const glob_1 = require("glob");
const path = __importStar(require("path"));
const readline_1 = __importDefault(require("readline"));
const assertions_1 = require("./assertions");
const cache_1 = require("./cache");
const cliState_1 = __importDefault(require("./cliState"));
const signal_1 = require("./database/signal");
const envars_1 = require("./envars");
const evaluatorHelpers_1 = require("./evaluatorHelpers");
const logger_1 = __importDefault(require("./logger"));
const prompt_1 = require("./models/prompt");
const azureUtil_1 = require("./providers/azureUtil");
const suggestions_1 = require("./suggestions");
const telemetry_1 = __importDefault(require("./telemetry"));
const types_1 = require("./types");
const writeToFile_1 = require("./util/exportToFile/writeToFile");
const invariant_1 = __importDefault(require("./util/invariant"));
const json_1 = require("./util/json");
const time_1 = require("./util/time");
const transform_1 = require("./util/transform");
exports.DEFAULT_MAX_CONCURRENCY = 4;
/**
 * Validates if a given prompt is allowed based on the provided list of allowed
 * prompt labels. Providers can be configured with a `prompts` attribute, which
 * corresponds to an array of prompt labels. Labels can either refer to a group
 * (for example from a file) or to individual prompts. If the attribute is
 * present, this function validates that the prompt labels fit the matching
 * criteria of the provider. Examples:
 *
 * - `prompts: ['examplePrompt']` matches `examplePrompt` exactly
 * - `prompts: ['exampleGroup:*']` matches any prompt that starts with `exampleGroup:`
 *
 * If no `prompts` attribute is present, all prompts are allowed by default.
 *
 * @param prompt - The prompt object to check.
 * @param allowedPrompts - The list of allowed prompt labels.
 * @returns Returns true if the prompt is allowed, false otherwise.
 */
function isAllowedPrompt(prompt, allowedPrompts) {
    return (!Array.isArray(allowedPrompts) ||
        allowedPrompts.includes(prompt.label) ||
        allowedPrompts.some((allowedPrompt) => prompt.label.startsWith(`${allowedPrompt}:`)));
}
function generateVarCombinations(vars) {
    const keys = Object.keys(vars);
    const combinations = [{}];
    for (const key of keys) {
        let values = [];
        if (typeof vars[key] === 'string' && vars[key].startsWith('file://')) {
            const filePath = vars[key].slice('file://'.length);
            const resolvedPath = path.resolve(cliState_1.default.basePath || '', filePath);
            const filePaths = (0, glob_1.globSync)(resolvedPath.replace(/\\/g, '/'));
            values = filePaths.map((path) => `file://${path}`);
            if (values.length === 0) {
                throw new Error(`No files found for variable ${key} at path ${resolvedPath}`);
            }
        }
        else {
            values = Array.isArray(vars[key]) ? vars[key] : [vars[key]];
        }
        // Check if it's an array but not a string array
        if (Array.isArray(vars[key]) && typeof vars[key][0] !== 'string') {
            values = [vars[key]];
        }
        const newCombinations = [];
        for (const combination of combinations) {
            for (const value of values) {
                newCombinations.push({ ...combination, [key]: value });
            }
        }
        combinations.length = 0;
        combinations.push(...newCombinations);
    }
    return combinations;
}
class Evaluator {
    constructor(testSuite, evalRecord, options) {
        this.testSuite = testSuite;
        this.evalRecord = evalRecord;
        this.options = options;
        this.stats = {
            successes: 0,
            failures: 0,
            errors: 0,
            tokenUsage: {
                total: 0,
                prompt: 0,
                completion: 0,
                cached: 0,
                numRequests: 0,
            },
        };
        this.conversations = {};
        this.registers = {};
        const jsonlFiles = Array.isArray(evalRecord.config.outputPath)
            ? evalRecord.config.outputPath.filter((p) => p.endsWith('.jsonl'))
            : evalRecord.config.outputPath?.endsWith('.jsonl')
                ? [evalRecord.config.outputPath]
                : [];
        this.fileWriters = jsonlFiles.map((p) => new writeToFile_1.JsonlFileWriter(p));
    }
    async runEval({ provider, prompt, // raw prompt
    test, delay, nunjucksFilters: filters, evaluateOptions, testIdx, promptIdx, }) {
        // Use the original prompt to set the label, not renderedPrompt
        const promptLabel = prompt.label;
        provider.delay ?? (provider.delay = delay ?? (0, envars_1.getEnvInt)('PROMPTFOO_DELAY_MS', 0));
        (0, invariant_1.default)(typeof provider.delay === 'number', `Provider delay should be set for ${provider.label}`);
        // Set up the special _conversation variable
        const vars = test.vars || {};
        const conversationKey = `${provider.label || provider.id()}:${prompt.id}${test.metadata?.conversationId ? `:${test.metadata.conversationId}` : ''}`;
        const usesConversation = prompt.raw.includes('_conversation');
        if (!(0, envars_1.getEnvBool)('PROMPTFOO_DISABLE_CONVERSATION_VAR') &&
            !test.options?.disableConversationVar &&
            usesConversation) {
            vars._conversation = this.conversations[conversationKey] || [];
        }
        // Overwrite vars with any saved register values
        Object.assign(vars, this.registers);
        // Render the prompt
        const renderedPrompt = await (0, evaluatorHelpers_1.renderPrompt)(prompt, vars, filters, provider);
        let renderedJson = undefined;
        try {
            renderedJson = JSON.parse(renderedPrompt);
        }
        catch { }
        const setup = {
            provider: {
                id: provider.id(),
                label: provider.label,
                config: provider.config,
            },
            prompt: {
                raw: renderedPrompt,
                label: promptLabel,
                config: prompt.config,
            },
            vars,
        };
        // Call the API
        let latencyMs = 0;
        try {
            const startTime = Date.now();
            let response = {
                output: '',
                tokenUsage: {},
                cost: 0,
                cached: false,
            };
            if (test.providerOutput) {
                response.output = test.providerOutput;
            }
            else {
                response = await (test.provider || provider).callApi(renderedPrompt, {
                    // Always included
                    vars,
                    // Part of these may be removed in python and script providers, but every Javascript provider gets them
                    prompt,
                    filters,
                    originalProvider: provider,
                    test,
                    // All of these are removed in python and script providers, but every Javascript provider gets them
                    logger: logger_1.default,
                    fetchWithCache: cache_1.fetchWithCache,
                    getCache: cache_1.getCache,
                }, {
                    includeLogProbs: test.assert?.some((a) => a.type === 'perplexity'),
                });
            }
            const endTime = Date.now();
            latencyMs = endTime - startTime;
            let conversationLastInput = undefined;
            if (renderedJson && Array.isArray(renderedJson)) {
                const lastElt = renderedJson[renderedJson.length - 1];
                // Use the `content` field if present (OpenAI chat format)
                conversationLastInput = lastElt?.content || lastElt;
            }
            this.conversations[conversationKey] = this.conversations[conversationKey] || [];
            this.conversations[conversationKey].push({
                prompt: renderedJson || renderedPrompt,
                input: conversationLastInput || renderedJson || renderedPrompt,
                output: response.output || '',
            });
            if (!response.cached && provider.delay > 0) {
                logger_1.default.debug(`Sleeping for ${provider.delay}ms`);
                await (0, time_1.sleep)(provider.delay);
            }
            const ret = {
                ...setup,
                response,
                success: false,
                failureReason: types_1.ResultFailureReason.NONE,
                score: 0,
                namedScores: {},
                latencyMs,
                cost: response.cost,
                metadata: response.metadata,
                promptIdx,
                testIdx,
                testCase: test,
                promptId: prompt.id || '',
            };
            if (response.error) {
                ret.error = response.error;
            }
            else if (response.output === null || response.output === undefined) {
                ret.success = false;
                ret.score = 0;
                ret.error = 'No output';
            }
            else {
                // Create a copy of response so we can potentially mutate it.
                const processedResponse = { ...response };
                const transforms = [
                    provider.transform, // Apply provider transform first
                    // NOTE: postprocess is deprecated. Use the first defined transform.
                    [test.options?.transform, test.options?.postprocess].find((s) => s),
                ]
                    .flat()
                    .filter((s) => typeof s === 'string');
                for (const t of transforms) {
                    processedResponse.output = await (0, transform_1.transform)(t, processedResponse.output, {
                        vars,
                        prompt,
                    });
                }
                (0, invariant_1.default)(processedResponse.output != null, 'Response output should not be null');
                const checkResult = await (0, assertions_1.runAssertions)({
                    prompt: renderedPrompt,
                    provider,
                    providerResponse: processedResponse,
                    test,
                    latencyMs: response.cached ? undefined : latencyMs,
                });
                if (!checkResult.pass) {
                    ret.error = checkResult.reason;
                    ret.failureReason = types_1.ResultFailureReason.ASSERT;
                }
                ret.success = checkResult.pass;
                ret.score = checkResult.score;
                ret.namedScores = checkResult.namedScores || {};
                if (checkResult.tokensUsed) {
                    this.stats.tokenUsage.total += checkResult.tokensUsed.total || 0;
                    this.stats.tokenUsage.prompt += checkResult.tokensUsed.prompt || 0;
                    this.stats.tokenUsage.completion += checkResult.tokensUsed.completion || 0;
                    this.stats.tokenUsage.cached += checkResult.tokensUsed.cached || 0;
                    this.stats.tokenUsage.numRequests += checkResult.tokensUsed.numRequests || 1;
                }
                ret.response = processedResponse;
                ret.gradingResult = checkResult;
            }
            // Update token usage stats
            if (response.tokenUsage) {
                this.stats.tokenUsage.total += response.tokenUsage.total || 0;
                this.stats.tokenUsage.prompt += response.tokenUsage.prompt || 0;
                this.stats.tokenUsage.completion += response.tokenUsage.completion || 0;
                this.stats.tokenUsage.cached += response.tokenUsage.cached || 0;
                this.stats.tokenUsage.numRequests += response.tokenUsage.numRequests || 1;
            }
            if (ret.success) {
                this.stats.successes++;
            }
            else if (ret.failureReason === types_1.ResultFailureReason.ERROR) {
                this.stats.errors++;
            }
            else {
                this.stats.failures++;
            }
            if (test.options?.storeOutputAs && ret.response?.output) {
                // Save the output in a register for later use
                this.registers[test.options.storeOutputAs] = ret.response.output;
            }
            return ret;
        }
        catch (err) {
            this.stats.errors++;
            return {
                ...setup,
                error: String(err) + '\n\n' + err.stack,
                success: false,
                failureReason: types_1.ResultFailureReason.ERROR,
                score: 0,
                namedScores: {},
                latencyMs,
                promptIdx,
                testIdx,
                testCase: test,
                promptId: prompt.id || '',
            };
        }
    }
    async evaluate() {
        const { testSuite, options } = this;
        const checkAbort = () => {
            if (options.abortSignal?.aborted) {
                throw new Error('Operation cancelled');
            }
        };
        // Add abort checks at key points
        checkAbort();
        const prompts = [];
        const assertionTypes = new Set();
        const rowsWithSelectBestAssertion = new Set();
        await (0, evaluatorHelpers_1.runExtensionHook)(testSuite.extensions, 'beforeAll', { suite: testSuite });
        if (options.generateSuggestions) {
            // TODO(ian): Move this into its own command/file
            logger_1.default.info(`Generating prompt variations...`);
            const { prompts: newPrompts, error } = await (0, suggestions_1.generatePrompts)(testSuite.prompts[0].raw, 1);
            if (error || !newPrompts) {
                throw new Error(`Failed to generate prompts: ${error}`);
            }
            logger_1.default.info(chalk_1.default.blue('Generated prompts:'));
            let numAdded = 0;
            for (const prompt of newPrompts) {
                logger_1.default.info('--------------------------------------------------------');
                logger_1.default.info(`${prompt}`);
                logger_1.default.info('--------------------------------------------------------');
                // Ask the user if they want to continue
                await new Promise((resolve) => {
                    const rl = readline_1.default.createInterface({
                        input: process.stdin,
                        output: process.stdout,
                    });
                    rl.question(`${chalk_1.default.blue('Do you want to test this prompt?')} (y/N): `, async (answer) => {
                        rl.close();
                        if (answer.toLowerCase().startsWith('y')) {
                            testSuite.prompts.push({ raw: prompt, label: prompt });
                            numAdded++;
                        }
                        else {
                            logger_1.default.info('Skipping this prompt.');
                        }
                        resolve(true);
                    });
                });
            }
            if (numAdded < 1) {
                logger_1.default.info(chalk_1.default.red('No prompts selected. Aborting.'));
                process.exitCode = 1;
                return this.evalRecord;
            }
        }
        // Split prompts by provider
        // Order matters - keep provider in outer loop to reduce need to swap models during local inference.
        for (const provider of testSuite.providers) {
            for (const prompt of testSuite.prompts) {
                // Check if providerPromptMap exists and if it contains the current prompt's label
                const providerKey = provider.label || provider.id();
                if (!isAllowedPrompt(prompt, testSuite.providerPromptMap?.[providerKey])) {
                    continue;
                }
                const completedPrompt = {
                    ...prompt,
                    id: (0, prompt_1.generateIdFromPrompt)(prompt),
                    provider: providerKey,
                    label: prompt.label,
                    metrics: {
                        score: 0,
                        testPassCount: 0,
                        testFailCount: 0,
                        testErrorCount: 0,
                        assertPassCount: 0,
                        assertFailCount: 0,
                        totalLatencyMs: 0,
                        tokenUsage: {
                            total: 0,
                            prompt: 0,
                            completion: 0,
                            cached: 0,
                            numRequests: 0,
                        },
                        namedScores: {},
                        namedScoresCount: {},
                        cost: 0,
                    },
                };
                prompts.push(completedPrompt);
            }
        }
        // Aggregate all vars across test cases
        let tests = testSuite.tests && testSuite.tests.length > 0
            ? testSuite.tests
            : testSuite.scenarios
                ? []
                : [
                    {
                    // Dummy test for cases when we're only comparing raw prompts.
                    },
                ];
        // Build scenarios and add to tests
        if (testSuite.scenarios && testSuite.scenarios.length > 0) {
            telemetry_1.default.recordAndSendOnce('feature_used', {
                feature: 'scenarios',
            });
            for (const scenario of testSuite.scenarios) {
                for (const data of scenario.config) {
                    // Merge defaultTest with scenario config
                    const scenarioTests = (scenario.tests || [
                        {
                        // Dummy test for cases when we're only comparing raw prompts.
                        },
                    ]).map((test) => {
                        return {
                            ...testSuite.defaultTest,
                            ...data,
                            ...test,
                            vars: {
                                ...testSuite.defaultTest?.vars,
                                ...data.vars,
                                ...test.vars,
                            },
                            options: {
                                ...testSuite.defaultTest?.options,
                                ...test.options,
                            },
                            assert: [
                                // defaultTest.assert is omitted because it will be added to each test case later
                                ...(data.assert || []),
                                ...(test.assert || []),
                            ],
                            metadata: {
                                ...testSuite.defaultTest?.metadata,
                                ...data.metadata,
                                ...test.metadata,
                            },
                        };
                    });
                    // Add scenario tests to tests
                    tests = tests.concat(scenarioTests);
                }
            }
        }
        (0, azureUtil_1.maybeEmitAzureOpenAiWarning)(testSuite, tests);
        // Prepare vars
        const varNames = new Set();
        const varsWithSpecialColsRemoved = [];
        const inputTransformDefault = testSuite?.defaultTest?.options?.transformVars;
        for (const testCase of tests) {
            testCase.vars = { ...testSuite.defaultTest?.vars, ...testCase?.vars };
            if (testCase.vars) {
                const varWithSpecialColsRemoved = {};
                const inputTransformForIndividualTest = testCase.options?.transformVars;
                const inputTransform = inputTransformForIndividualTest || inputTransformDefault;
                if (inputTransform) {
                    const transformContext = {
                        prompt: {},
                        uuid: (0, crypto_1.randomUUID)(),
                    };
                    const transformedVars = await (0, transform_1.transform)(inputTransform, testCase.vars, transformContext, true, transform_1.TransformInputType.VARS);
                    (0, invariant_1.default)(typeof transformedVars === 'object', 'Transform function did not return a valid object');
                    testCase.vars = { ...testCase.vars, ...transformedVars };
                }
                for (const varName of Object.keys(testCase.vars)) {
                    varNames.add(varName);
                    varWithSpecialColsRemoved[varName] = testCase.vars[varName];
                }
                varsWithSpecialColsRemoved.push(varWithSpecialColsRemoved);
            }
        }
        // Set up eval cases
        const runEvalOptions = [];
        let testIdx = 0;
        for (let index = 0; index < tests.length; index++) {
            const testCase = tests[index];
            (0, invariant_1.default)(Array.isArray(testSuite.defaultTest?.assert || []), `defaultTest.assert is not an array in test case #${index + 1}`);
            (0, invariant_1.default)(Array.isArray(testCase.assert || []), `testCase.assert is not an array in test case #${index + 1}`);
            // Handle default properties
            testCase.assert = [...(testSuite.defaultTest?.assert || []), ...(testCase.assert || [])];
            testCase.threshold = testCase.threshold ?? testSuite.defaultTest?.threshold;
            testCase.options = { ...testSuite.defaultTest?.options, ...testCase.options };
            testCase.metadata = { ...testSuite.defaultTest?.metadata, ...testCase.metadata };
            testCase.provider = testCase.provider || testSuite.defaultTest?.provider;
            const prependToPrompt = testCase.options?.prefix || testSuite.defaultTest?.options?.prefix || '';
            const appendToPrompt = testCase.options?.suffix || testSuite.defaultTest?.options?.suffix || '';
            // Finalize test case eval
            const varCombinations = (0, envars_1.getEnvBool)('PROMPTFOO_DISABLE_VAR_EXPANSION') || testCase.options.disableVarExpansion
                ? [testCase.vars]
                : generateVarCombinations(testCase.vars || {});
            const numRepeat = this.options.repeat || 1;
            for (let repeatIndex = 0; repeatIndex < numRepeat; repeatIndex++) {
                for (const vars of varCombinations) {
                    let promptIdx = 0;
                    // Order matters - keep provider in outer loop to reduce need to swap models during local inference.
                    for (const provider of testSuite.providers) {
                        for (const prompt of testSuite.prompts) {
                            const providerKey = provider.label || provider.id();
                            if (!isAllowedPrompt(prompt, testSuite.providerPromptMap?.[providerKey])) {
                                continue;
                            }
                            runEvalOptions.push({
                                delay: options.delay || 0,
                                provider,
                                prompt: {
                                    ...prompt,
                                    raw: prependToPrompt + prompt.raw + appendToPrompt,
                                },
                                test: { ...testCase, vars, options: testCase.options },
                                nunjucksFilters: testSuite.nunjucksFilters,
                                testIdx,
                                promptIdx,
                                repeatIndex,
                                evaluateOptions: options,
                            });
                            promptIdx++;
                        }
                    }
                    testIdx++;
                }
            }
        }
        // Determine run parameters
        let concurrency = options.maxConcurrency || exports.DEFAULT_MAX_CONCURRENCY;
        if (concurrency > 1) {
            const usesConversation = prompts.some((p) => p.raw.includes('_conversation'));
            const usesStoreOutputAs = tests.some((t) => t.options?.storeOutputAs);
            if (usesConversation) {
                logger_1.default.info(`Setting concurrency to 1 because the ${chalk_1.default.cyan('_conversation')} variable is used.`);
                concurrency = 1;
            }
            else if (usesStoreOutputAs) {
                logger_1.default.info(`Setting concurrency to 1 because storeOutputAs is used.`);
                concurrency = 1;
            }
        }
        // Actually run the eval
        let numComplete = 0;
        const processEvalStep = async (evalStep, index) => {
            if (typeof index !== 'number') {
                throw new Error('Expected index to be a number');
            }
            await (0, evaluatorHelpers_1.runExtensionHook)(testSuite.extensions, 'beforeEach', {
                test: evalStep.test,
            });
            const row = await this.runEval(evalStep);
            if (evalStep.test.assert?.some((a) => a.type === 'select-best')) {
                rowsWithSelectBestAssertion.add(row.testIdx);
            }
            for (const assert of evalStep.test.assert || []) {
                if (assert.type) {
                    assertionTypes.add(assert.type);
                }
            }
            numComplete++;
            if (options.progressCallback) {
                options.progressCallback(this.evalRecord.results.length, runEvalOptions.length, index, evalStep);
            }
            try {
                await this.evalRecord.addResult(row, evalStep.test);
            }
            catch (error) {
                logger_1.default.error(`Error saving result: ${error} ${(0, json_1.safeJsonStringify)(row)}`);
            }
            for (const writer of this.fileWriters) {
                await writer.write(row);
            }
            const { promptIdx } = row;
            const metrics = prompts[promptIdx].metrics;
            (0, invariant_1.default)(metrics, 'Expected prompt.metrics to be set');
            metrics.score += row.score;
            for (const [key, value] of Object.entries(row.namedScores)) {
                metrics.namedScores[key] = (metrics.namedScores[key] || 0) + value;
                metrics.namedScoresCount[key] = (metrics.namedScoresCount[key] || 0) + 1;
            }
            if (testSuite.derivedMetrics) {
                const math = await Promise.resolve().then(() => __importStar(require('mathjs'))); // TODO: move this
                for (const metric of testSuite.derivedMetrics) {
                    if (metrics.namedScores[metric.name] === undefined) {
                        metrics.namedScores[metric.name] = 0;
                    }
                    try {
                        if (typeof metric.value === 'function') {
                            metrics.namedScores[metric.name] = metric.value(metrics.namedScores, evalStep);
                        }
                        else {
                            const evaluatedValue = math.evaluate(metric.value, metrics.namedScores);
                            metrics.namedScores[metric.name] = evaluatedValue;
                        }
                    }
                    catch (error) {
                        logger_1.default.debug(`Could not evaluate derived metric '${metric.name}': ${error.message}`);
                    }
                }
            }
            metrics.testPassCount += row.success ? 1 : 0;
            if (!row.success) {
                if (row.failureReason === types_1.ResultFailureReason.ERROR) {
                    metrics.testErrorCount += 1;
                }
                else {
                    metrics.testFailCount += 1;
                }
            }
            metrics.assertPassCount +=
                row.gradingResult?.componentResults?.filter((r) => r.pass).length || 0;
            metrics.assertFailCount +=
                row.gradingResult?.componentResults?.filter((r) => !r.pass).length || 0;
            metrics.totalLatencyMs += row.latencyMs || 0;
            metrics.tokenUsage.cached =
                (metrics.tokenUsage.cached || 0) + (row.response?.tokenUsage?.cached || 0);
            metrics.tokenUsage.completion =
                (metrics.tokenUsage.completion || 0) + (row.response?.tokenUsage?.completion || 0);
            metrics.tokenUsage.prompt =
                (metrics.tokenUsage.prompt || 0) + (row.response?.tokenUsage?.prompt || 0);
            metrics.tokenUsage.total =
                (metrics.tokenUsage.total || 0) + (row.response?.tokenUsage?.total || 0);
            metrics.tokenUsage.numRequests =
                (metrics.tokenUsage.numRequests || 0) + (row.response?.tokenUsage?.numRequests || 1);
            metrics.cost += row.cost || 0;
            await (0, evaluatorHelpers_1.runExtensionHook)(testSuite.extensions, 'afterEach', {
                test: evalStep.test,
                result: row,
            });
        };
        // Set up main progress bars
        let multibar;
        let multiProgressBars = [];
        const originalProgressCallback = this.options.progressCallback;
        const isWebUI = Boolean(cliState_1.default.webUI);
        this.options.progressCallback = (completed, total, index, evalStep) => {
            if (originalProgressCallback) {
                originalProgressCallback(completed, total, index, evalStep);
            }
            if (isWebUI) {
                const provider = evalStep.provider.label || evalStep.provider.id();
                const vars = Object.entries(evalStep.test.vars || {})
                    .map(([k, v]) => `${k}=${v}`)
                    .join(' ')
                    .slice(0, 50)
                    .replace(/\n/g, ' ');
                logger_1.default.info(`[${numComplete}/${total}] Running ${provider} with vars: ${vars}`);
            }
            else if (multibar && evalStep) {
                const threadIndex = index % concurrency;
                const progressbar = multiProgressBars[threadIndex];
                progressbar.increment({
                    provider: evalStep.provider.label || evalStep.provider.id(),
                    prompt: evalStep.prompt.raw.slice(0, 10).replace(/\n/g, ' '),
                    vars: Object.entries(evalStep.test.vars || {})
                        .map(([k, v]) => `${k}=${v}`)
                        .join(' ')
                        .slice(0, 10)
                        .replace(/\n/g, ' '),
                });
            }
            else {
                logger_1.default.debug(`Eval #${index + 1} complete (${numComplete} of ${runEvalOptions.length})`);
            }
        };
        const createMultiBars = async (evalOptions) => {
            // Only create progress bars if not in web UI mode
            if (isWebUI) {
                return;
            }
            const cliProgress = await Promise.resolve().then(() => __importStar(require('cli-progress')));
            multibar = new cliProgress.MultiBar({
                format: '[{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | {provider} "{prompt}" {vars}',
                hideCursor: true,
            }, cliProgress.Presets.shades_classic);
            const stepsPerThread = Math.floor(evalOptions.length / concurrency);
            const remainingSteps = evalOptions.length % concurrency;
            multiProgressBars = [];
            for (let i = 0; i < concurrency; i++) {
                const totalSteps = i < remainingSteps ? stepsPerThread + 1 : stepsPerThread;
                if (totalSteps > 0) {
                    const progressbar = multibar.create(totalSteps, 0, {
                        provider: '',
                        prompt: '',
                        vars: '',
                    });
                    multiProgressBars.push(progressbar);
                }
            }
        };
        // Run the evals
        if (this.options.showProgressBar) {
            await createMultiBars(runEvalOptions);
        }
        // Separate serial and concurrent eval options
        const serialRunEvalOptions = [];
        const concurrentRunEvalOptions = [];
        for (const evalOption of runEvalOptions) {
            if (evalOption.test.options?.runSerially) {
                serialRunEvalOptions.push(evalOption);
            }
            else {
                concurrentRunEvalOptions.push(evalOption);
            }
        }
        if (serialRunEvalOptions.length > 0) {
            // Run serial evaluations first
            logger_1.default.info(`Running ${serialRunEvalOptions.length} serial evaluations...`);
            for (const evalStep of serialRunEvalOptions) {
                if (isWebUI) {
                    const provider = evalStep.provider.label || evalStep.provider.id();
                    const vars = Object.entries(evalStep.test.vars || {})
                        .map(([k, v]) => `${k}=${v}`)
                        .join(' ')
                        .slice(0, 50)
                        .replace(/\n/g, ' ');
                    logger_1.default.info(`[${numComplete}/${serialRunEvalOptions.length}] Running ${provider} with vars: ${vars}`);
                }
                await processEvalStep(evalStep, serialRunEvalOptions.indexOf(evalStep));
            }
        }
        // Then run concurrent evaluations
        logger_1.default.info(`Running ${concurrentRunEvalOptions.length} concurrent evaluations with up to ${concurrency} threads...`);
        await async_1.default.forEachOfLimit(concurrentRunEvalOptions, concurrency, async (evalStep, index) => {
            checkAbort();
            await processEvalStep(evalStep, index);
        });
        // Do we have to run comparisons between row outputs?
        const compareRowsCount = rowsWithSelectBestAssertion.size;
        let progressBar;
        if (compareRowsCount > 0 && multibar && !isWebUI) {
            progressBar = multibar.create(compareRowsCount, 0, {
                provider: 'Running model-graded comparisons',
                prompt: '',
                vars: '',
            });
        }
        let compareCount = 0;
        for (const testIdx of rowsWithSelectBestAssertion) {
            compareCount++;
            if (isWebUI) {
                logger_1.default.info(`Running model-graded comparison ${compareCount} of ${compareRowsCount}...`);
            }
            const resultsToCompare = this.evalRecord.persisted
                ? await this.evalRecord.fetchResultsByTestIdx(testIdx)
                : this.evalRecord.results.filter((r) => r.testIdx === testIdx);
            if (resultsToCompare.length === 0) {
                logger_1.default.warn(`Expected results to be found for test index ${testIdx}`);
                continue;
            }
            const compareAssertion = resultsToCompare[0].testCase.assert?.find((a) => a.type === 'select-best');
            if (compareAssertion) {
                const outputs = resultsToCompare.map((r) => r.response?.output || '');
                const gradingResults = await (0, assertions_1.runCompareAssertion)(resultsToCompare[0].testCase, compareAssertion, outputs);
                for (let index = 0; index < resultsToCompare.length; index++) {
                    const result = resultsToCompare[index];
                    const gradingResult = gradingResults[index];
                    if (result.gradingResult) {
                        result.gradingResult.tokensUsed = result.gradingResult.tokensUsed || {
                            total: 0,
                            prompt: 0,
                            completion: 0,
                        };
                        result.gradingResult.tokensUsed = result.gradingResult.tokensUsed || {
                            total: 0,
                            prompt: 0,
                            completion: 0,
                        };
                        result.gradingResult.tokensUsed.total =
                            (result.gradingResult.tokensUsed.total || 0) + (gradingResult.tokensUsed?.total || 0);
                        result.gradingResult.tokensUsed.prompt =
                            (result.gradingResult.tokensUsed.prompt || 0) +
                                (gradingResult.tokensUsed?.prompt || 0);
                        result.gradingResult.tokensUsed.completion =
                            (result.gradingResult.tokensUsed.completion || 0) +
                                (gradingResult.tokensUsed?.completion || 0);
                        result.success = result.gradingResult.pass =
                            result.gradingResult.pass && gradingResult.pass;
                        if (!gradingResult.pass) {
                            // Failure overrides the reason and the score
                            result.gradingResult.reason = gradingResult.reason;
                            result.score = result.gradingResult.score = gradingResult.score;
                        }
                        if (!result.gradingResult.componentResults) {
                            result.gradingResult.componentResults = [];
                        }
                        result.gradingResult.componentResults.push(gradingResult);
                    }
                    else {
                        result.gradingResult = gradingResult;
                    }
                    if (this.evalRecord.persisted) {
                        await result.save();
                    }
                }
                if (progressBar) {
                    progressBar.increment({
                        prompt: resultsToCompare[0].prompt.raw.slice(0, 10).replace(/\n/g, ''),
                    });
                }
                else if (!isWebUI) {
                    logger_1.default.debug(`Model-graded comparison #${compareCount} of ${compareRowsCount} complete`);
                }
            }
        }
        await this.evalRecord.addPrompts(prompts);
        // Finish up
        if (multibar) {
            multibar.stop();
        }
        if (progressBar) {
            progressBar.stop();
        }
        await (0, evaluatorHelpers_1.runExtensionHook)(testSuite.extensions, 'afterAll', {
            prompts: this.evalRecord.prompts,
            suite: testSuite,
        });
        telemetry_1.default.record('eval_ran', {
            numPrompts: prompts.length,
            numTests: prompts.reduce((acc, p) => acc +
                (p.metrics?.testPassCount || 0) +
                (p.metrics?.testFailCount || 0) +
                (p.metrics?.testErrorCount || 0), 0),
            numVars: varNames.size,
            numProviders: testSuite.providers.length,
            numRepeat: options.repeat || 1,
            providerPrefixes: Array.from(new Set(testSuite.providers.map((p) => {
                const idParts = p.id().split(':');
                return idParts.length > 1 ? idParts[0] : 'unknown';
            }))).sort(),
            assertionTypes: Array.from(assertionTypes).sort(),
            eventSource: options.eventSource || 'default',
            ci: (0, envars_1.isCI)(),
            hasAnyPass: this.evalRecord.prompts.some((p) => p.metrics?.testPassCount && p.metrics.testPassCount > 0),
            // FIXME(ian): Does this work?  I think redteam is only on the config, not testSuite.
            // isRedteam: Boolean(testSuite.redteam),
        });
        // Update database signal file after all results are written
        (0, signal_1.updateSignalFile)();
        return this.evalRecord;
    }
}
function evaluate(testSuite, evalRecord, options) {
    const ev = new Evaluator(testSuite, evalRecord, options);
    return ev.evaluate();
}
//# sourceMappingURL=evaluator.js.map