#!/usr/bin/env node
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const readline_1 = __importDefault(require("readline"));
const chalk_1 = __importDefault(require("chalk"));
const chokidar_1 = __importDefault(require("chokidar"));
const js_yaml_1 = __importDefault(require("js-yaml"));
const commander_1 = require("commander");
const cliState_1 = __importDefault(require("./cliState"));
const telemetry_1 = __importDefault(require("./telemetry"));
const logger_1 = __importStar(require("./logger"));
const assertions_1 = require("./assertions");
const providers_1 = require("./providers");
const evaluator_1 = require("./evaluator");
const prompts_1 = require("./prompts");
const testCases_1 = require("./testCases");
const redteam_1 = require("./redteam");
const constants_1 = require("./redteam/constants");
const util_1 = require("./util");
const onboarding_1 = require("./onboarding");
const cache_1 = require("./cache");
const esm_1 = require("./esm");
const server_1 = require("./web/server");
const updates_1 = require("./updates");
const feedback_1 = require("./feedback");
const list_1 = require("./commands/list");
const show_1 = require("./commands/show");
const delete_1 = require("./commands/delete");
const import_1 = require("./commands/import");
const export_1 = require("./commands/export");
const config_1 = require("./commands/config");
const table_1 = require("./table");
const share_1 = require("./share");
const filterTests_1 = require("./commands/eval/filterTests");
const validateAssertions_1 = require("./assertions/validateAssertions");
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
const dedent_1 = __importDefault(require("dedent"));
async function resolveConfigs(cmdObj, defaultConfig) {
    // Config parsing
    let fileConfig = {};
    const configPaths = cmdObj.config;
    if (configPaths) {
        fileConfig = await (0, util_1.readConfigs)(configPaths);
    }
    // Standalone assertion mode
    if (cmdObj.assertions) {
        if (!cmdObj.modelOutputs) {
            logger_1.default.error(chalk_1.default.red('You must provide --model-outputs when using --assertions'));
            process.exit(1);
        }
        const modelOutputs = JSON.parse(fs_1.default.readFileSync(path_1.default.join(process.cwd(), cmdObj.modelOutputs), 'utf8'));
        const assertions = await (0, assertions_1.readAssertions)(cmdObj.assertions);
        fileConfig.prompts = ['{{output}}'];
        fileConfig.providers = ['echo'];
        fileConfig.tests = modelOutputs.map((output) => {
            if (typeof output === 'string') {
                return {
                    vars: {
                        output,
                    },
                    assert: assertions,
                };
            }
            return {
                vars: {
                    output: output.output,
                    ...(output.tags === undefined ? {} : { tags: output.tags.join(', ') }),
                },
                assert: assertions,
            };
        });
    }
    // Use basepath in cases where path was supplied in the config file
    const basePath = configPaths ? path_1.default.dirname(configPaths[0]) : '';
    const defaultTestRaw = fileConfig.defaultTest || defaultConfig.defaultTest;
    const config = {
        description: fileConfig.description || defaultConfig.description,
        prompts: cmdObj.prompts || fileConfig.prompts || defaultConfig.prompts || [],
        providers: cmdObj.providers || fileConfig.providers || defaultConfig.providers || [],
        tests: cmdObj.tests || cmdObj.vars || fileConfig.tests || defaultConfig.tests || [],
        scenarios: fileConfig.scenarios || defaultConfig.scenarios,
        env: fileConfig.env || defaultConfig.env,
        sharing: process.env.PROMPTFOO_DISABLE_SHARING === '1'
            ? false
            : fileConfig.sharing ?? defaultConfig.sharing ?? true,
        defaultTest: defaultTestRaw ? await (0, testCases_1.readTest)(defaultTestRaw, basePath) : undefined,
        derivedMetrics: fileConfig.derivedMetrics || defaultConfig.derivedMetrics,
        outputPath: cmdObj.output || fileConfig.outputPath || defaultConfig.outputPath,
        metadata: fileConfig.metadata || defaultConfig.metadata,
    };
    // Validation
    if (!config.prompts || config.prompts.length === 0) {
        logger_1.default.error(chalk_1.default.red('You must provide at least 1 prompt'));
        process.exit(1);
    }
    if (!config.providers || config.providers.length === 0) {
        logger_1.default.error(chalk_1.default.red('You must specify at least 1 provider (for example, openai:gpt-3.5-turbo)'));
        process.exit(1);
    }
    // Parse prompts, providers, and tests
    const parsedPrompts = await (0, prompts_1.readPrompts)(config.prompts, cmdObj.prompts ? undefined : basePath);
    const parsedProviders = await (0, providers_1.loadApiProviders)(config.providers, {
        env: config.env,
        basePath,
    });
    const parsedTests = await (0, testCases_1.readTests)(config.tests || [], cmdObj.tests ? undefined : basePath);
    // Parse testCases for each scenario
    if (fileConfig.scenarios) {
        for (const scenario of fileConfig.scenarios) {
            const parsedScenarioTests = await (0, testCases_1.readTests)(scenario.tests, cmdObj.tests ? undefined : basePath);
            scenario.tests = parsedScenarioTests;
            const filteredTests = await (0, filterTests_1.filterTests)({
                ...scenario,
                providers: parsedProviders,
                prompts: parsedPrompts,
            }, {
                firstN: cmdObj.filterFirstN,
                pattern: cmdObj.filterPattern,
                failing: cmdObj.filterFailing,
            });
            (0, tiny_invariant_1.default)(filteredTests, 'filteredTests are undefined');
            scenario.tests = filteredTests;
        }
    }
    const parsedProviderPromptMap = (0, prompts_1.readProviderPromptMap)(config, parsedPrompts);
    if (parsedPrompts.length === 0) {
        logger_1.default.error(chalk_1.default.red('No prompts found'));
        process.exit(1);
    }
    const defaultTest = {
        options: {
            prefix: cmdObj.promptPrefix,
            suffix: cmdObj.promptSuffix,
            provider: cmdObj.grader,
            // rubricPrompt
            ...(config.defaultTest?.options || {}),
        },
        ...config.defaultTest,
    };
    const testSuite = {
        description: config.description,
        prompts: parsedPrompts,
        providers: parsedProviders,
        providerPromptMap: parsedProviderPromptMap,
        tests: parsedTests,
        scenarios: config.scenarios,
        defaultTest,
        derivedMetrics: config.derivedMetrics,
        nunjucksFilters: await (0, util_1.readFilters)(fileConfig.nunjucksFilters || defaultConfig.nunjucksFilters || {}),
    };
    if (testSuite.tests) {
        (0, validateAssertions_1.validateAssertions)(testSuite.tests);
    }
    return { config, testSuite, basePath };
}
async function main() {
    await (0, updates_1.checkForUpdates)();
    const pwd = process.cwd();
    const potentialPaths = [
        path_1.default.join(pwd, 'promptfooconfig.js'),
        path_1.default.join(pwd, 'promptfooconfig.json'),
        path_1.default.join(pwd, 'promptfooconfig.yaml'),
        path_1.default.join(pwd, 'promptfooconfig.yml'),
    ];
    let defaultConfig = {};
    let defaultConfigPath;
    for (const _path of potentialPaths) {
        const maybeConfig = await (0, util_1.maybeReadConfig)(_path);
        if (maybeConfig) {
            defaultConfig = maybeConfig;
            defaultConfigPath = _path;
            break;
        }
    }
    const evaluateOptions = {};
    if (defaultConfig.evaluateOptions) {
        evaluateOptions.generateSuggestions = defaultConfig.evaluateOptions.generateSuggestions;
        evaluateOptions.maxConcurrency = defaultConfig.evaluateOptions.maxConcurrency;
        evaluateOptions.showProgressBar = defaultConfig.evaluateOptions.showProgressBar;
        evaluateOptions.interactiveProviders = defaultConfig.evaluateOptions.interactiveProviders;
    }
    const program = new commander_1.Command();
    program.option('--version', 'Print version', () => {
        const packageJson = JSON.parse(fs_1.default.readFileSync(path_1.default.join((0, esm_1.getDirectory)(), '../package.json'), 'utf8'));
        logger_1.default.info(packageJson.version);
        process.exit(0);
    });
    program
        .command('init [directory]')
        .description('Initialize project with dummy files')
        .option('--no-interactive', 'Run in interactive mode')
        .action(async (directory, cmdObj) => {
        telemetry_1.default.maybeShowNotice();
        telemetry_1.default.record('command_used', {
            name: 'init - started',
        });
        const details = await (0, onboarding_1.createDummyFiles)(directory, cmdObj.interactive);
        telemetry_1.default.record('command_used', {
            ...details,
            name: 'init',
        });
        await telemetry_1.default.send();
    });
    program
        .command('view [directory]')
        .description('Start browser ui')
        .option('-p, --port <number>', 'Port number', '15500')
        .option('-y, --yes', 'Skip confirmation and auto-open the URL')
        .option('-n, --no', 'Skip confirmation and do not open the URL')
        .option('--api-base-url <url>', 'Base URL for viewer API calls')
        .option('--filter-description <pattern>', 'Filter evals by description using a regex pattern')
        .option('--env-file <path>', 'Path to .env file')
        .action(async (directory, cmdObj) => {
        (0, util_1.setupEnv)(cmdObj.envFile);
        telemetry_1.default.maybeShowNotice();
        telemetry_1.default.record('command_used', {
            name: 'view',
        });
        await telemetry_1.default.send();
        if (directory) {
            (0, util_1.setConfigDirectoryPath)(directory);
        }
        // Block indefinitely on server
        const browserBehavior = cmdObj.yes
            ? server_1.BrowserBehavior.OPEN
            : cmdObj.no
                ? server_1.BrowserBehavior.SKIP
                : server_1.BrowserBehavior.ASK;
        await (0, server_1.startServer)(cmdObj.port, cmdObj.apiBaseUrl, browserBehavior, cmdObj.filterDescription);
    });
    program
        .command('share')
        .description('Create a shareable URL of your most recent eval')
        .option('-y, --yes', 'Skip confirmation')
        .option('--env-file <path>', 'Path to .env file')
        .action(async (cmdObj) => {
        (0, util_1.setupEnv)(cmdObj.envFile);
        telemetry_1.default.maybeShowNotice();
        telemetry_1.default.record('command_used', {
            name: 'share',
        });
        await telemetry_1.default.send();
        const createPublicUrl = async () => {
            const latestResults = await (0, util_1.readLatestResults)();
            if (!latestResults) {
                logger_1.default.error('Could not load results. Do you need to run `promptfoo eval` first?');
                process.exit(1);
            }
            const url = await (0, share_1.createShareableUrl)(latestResults.results, latestResults.config);
            logger_1.default.info(`View results: ${chalk_1.default.greenBright.bold(url)}`);
        };
        if (cmdObj.yes || process.env.PROMPTFOO_DISABLE_SHARE_WARNING) {
            createPublicUrl();
        }
        else {
            const reader = readline_1.default.createInterface({
                input: process.stdin,
                output: process.stdout,
            });
            reader.question('Create a private shareable URL of your most recent eval?\n\nTo proceed, please confirm [Y/n] ', async function (answer) {
                if (answer.toLowerCase() !== 'yes' && answer.toLowerCase() !== 'y' && answer !== '') {
                    reader.close();
                    process.exit(1);
                }
                reader.close();
                createPublicUrl();
            });
        }
    });
    program
        .command('cache')
        .description('Manage cache')
        .command('clear')
        .description('Clear cache')
        .option('--env-file <path>', 'Path to .env file')
        .action(async (cmdObj) => {
        (0, util_1.setupEnv)(cmdObj.envFile);
        telemetry_1.default.maybeShowNotice();
        logger_1.default.info('Clearing cache...');
        await (0, cache_1.clearCache)();
        (0, util_1.cleanupOldFileResults)(0);
        telemetry_1.default.record('command_used', {
            name: 'cache_clear',
        });
        await telemetry_1.default.send();
    });
    program
        .command('feedback [message]')
        .description('Send feedback to the promptfoo developers')
        .action((message) => {
        (0, feedback_1.gatherFeedback)(message);
    });
    const generateCommand = program.command('generate').description('Generate synthetic data');
    generateCommand
        .command('dataset')
        .description('Generate test cases')
        .option('-i, --instructions [instructions]', 'Additional instructions to follow while generating test cases')
        .option('-c, --config [path]', 'Path to configuration file. Defaults to promptfooconfig.yaml')
        .option('-o, --output [path]', 'Path to output file')
        .option('-w, --write', 'Write results to promptfoo configuration file')
        .option('--numPersonas <number>', 'Number of personas to generate', '5')
        .option('--numTestCasesPerPersona <number>', 'Number of test cases per persona', '3')
        .option('--no-cache', 'Do not read or write results to disk cache', false)
        .option('--env-file <path>', 'Path to .env file')
        .action(async (options) => {
        (0, util_1.setupEnv)(options.envFile);
        if (!options.cache) {
            logger_1.default.info('Cache is disabled.');
            (0, cache_1.disableCache)();
        }
        let testSuite;
        const configPath = options.config || defaultConfigPath;
        if (configPath) {
            const resolved = await resolveConfigs({
                config: [configPath],
            }, defaultConfig);
            testSuite = resolved.testSuite;
        }
        else {
            throw new Error('Could not find config file. Please use `--config`');
        }
        const startTime = Date.now();
        telemetry_1.default.record('command_used', {
            name: 'generate_dataset - started',
            numPrompts: testSuite.prompts.length,
            numTestsExisting: (testSuite.tests || []).length,
        });
        await telemetry_1.default.send();
        const results = await (0, testCases_1.synthesizeFromTestSuite)(testSuite, {
            instructions: options.instructions,
            numPersonas: parseInt(options.numPersonas, 10),
            numTestCasesPerPersona: parseInt(options.numTestCasesPerPersona, 10),
        });
        const configAddition = { tests: results.map((result) => ({ vars: result })) };
        const yamlString = js_yaml_1.default.dump(configAddition);
        if (options.output) {
            fs_1.default.writeFileSync(options.output, yamlString);
            (0, util_1.printBorder)();
            logger_1.default.info(`Wrote ${results.length} new test cases to ${options.output}`);
            (0, util_1.printBorder)();
        }
        else {
            (0, util_1.printBorder)();
            logger_1.default.info('New test Cases');
            (0, util_1.printBorder)();
            logger_1.default.info(yamlString);
        }
        (0, util_1.printBorder)();
        if (options.write && configPath) {
            const existingConfig = js_yaml_1.default.load(fs_1.default.readFileSync(configPath, 'utf8'));
            existingConfig.tests = [...(existingConfig.tests || []), ...configAddition.tests];
            fs_1.default.writeFileSync(configPath, js_yaml_1.default.dump(existingConfig));
            logger_1.default.info(`Wrote ${results.length} new test cases to ${configPath}`);
        }
        else {
            logger_1.default.info(`Copy the above test cases or run ${chalk_1.default.greenBright('promptfoo generate dataset --write')} to write directly to the config`);
        }
        telemetry_1.default.record('command_used', {
            name: 'generate_dataset',
            numPrompts: testSuite.prompts.length,
            numTestsExisting: (testSuite.tests || []).length,
            numTestsGenerated: results.length,
            duration: Math.round((Date.now() - startTime) / 1000),
        });
        await telemetry_1.default.send();
    });
    generateCommand
        .command('redteam')
        .description('Generate adversarial test cases')
        .option('-c, --config [path]', 'Path to configuration file. Defaults to promptfooconfig.yaml')
        .option('-o, --output [path]', 'Path to output file')
        .option('-w, --write', 'Write results to promptfoo configuration file')
        .option('--purpose <purpose>', 'Set the system purpose. If not set, the system purpose will be inferred from the config file')
        .option('--provider <provider>', `Provider to use for generating adversarial tests. Defaults to: ${constants_1.REDTEAM_MODEL}`)
        .option('--injectVar <varname>', 'Override the variable to inject user input into the prompt. If not set, the variable will default to {{query}}')
        .option('--plugins <plugins>', (0, dedent_1.default) `Comma-separated list of plugins to use. Defaults to:
        \n- ${Array.from(redteam_1.DEFAULT_PLUGINS).sort().join('\n- ')}\n\n
    `, (val) => val.split(',').map((x) => x.trim()))
        .option('--add-plugins <plugins>', (0, dedent_1.default) `Comma-separated list of plugins to run in addition to the default plugins:
        \n- ${redteam_1.ADDITIONAL_PLUGINS.sort().join('\n- ')}\n\n
      `, (val) => val.split(',').map((x) => x.trim()))
        .option('--no-cache', 'Do not read or write results to disk cache', false)
        .option('--env-file <path>', 'Path to .env file')
        .action(async ({ addPlugins, cache, config, envFile, injectVar, output, plugins, provider, purpose, write, }) => {
        (0, util_1.setupEnv)(envFile);
        if (!cache) {
            logger_1.default.info('Cache is disabled.');
            (0, cache_1.disableCache)();
        }
        let testSuite;
        const configPath = config || defaultConfigPath;
        if (configPath) {
            const resolved = await resolveConfigs({
                config: [configPath],
            }, defaultConfig);
            testSuite = resolved.testSuite;
        }
        else {
            throw new Error('Could not find config file. Please use `--config`');
        }
        const startTime = Date.now();
        telemetry_1.default.record('command_used', {
            name: 'generate redteam - started',
            numPrompts: testSuite.prompts.length,
            numTestsExisting: (testSuite.tests || []).length,
        });
        await telemetry_1.default.send();
        const redteamTests = await (0, redteam_1.synthesizeFromTestSuite)(testSuite, {
            purpose,
            injectVar,
            plugins: addPlugins && addPlugins.length > 0
                ? Array.from(plugins || redteam_1.DEFAULT_PLUGINS).concat(addPlugins)
                : plugins,
            provider,
        });
        if (output) {
            const existingYaml = js_yaml_1.default.load(fs_1.default.readFileSync(configPath, 'utf8'));
            const updatedYaml = {
                ...existingYaml,
                tests: redteamTests,
                metadata: {
                    ...existingYaml.metadata,
                    redteam: true,
                },
            };
            fs_1.default.writeFileSync(output, js_yaml_1.default.dump(updatedYaml, { skipInvalid: true }));
            (0, util_1.printBorder)();
            logger_1.default.info(`Wrote ${redteamTests.length} new test cases to ${output}`);
            (0, util_1.printBorder)();
        }
        else if (write && configPath) {
            const existingConfig = js_yaml_1.default.load(fs_1.default.readFileSync(configPath, 'utf8'));
            existingConfig.tests = [...(existingConfig.tests || []), ...redteamTests];
            fs_1.default.writeFileSync(configPath, js_yaml_1.default.dump(existingConfig));
            logger_1.default.info(`Wrote ${redteamTests.length} new test cases to ${configPath}`);
        }
        else {
            logger_1.default.info(js_yaml_1.default.dump(redteamTests, { skipInvalid: true }));
        }
        telemetry_1.default.record('command_used', {
            name: 'generate redteam',
            numPrompts: testSuite.prompts.length,
            numTestsExisting: (testSuite.tests || []).length,
            numTestsGenerated: redteamTests.length,
            duration: Math.round((Date.now() - startTime) / 1000),
        });
        await telemetry_1.default.send();
    });
    program
        .command('eval')
        .description('Evaluate prompts')
        .option('-p, --prompts <paths...>', 'Paths to prompt files (.txt)')
        .option('-r, --providers <name or path...>', 'One of: openai:chat, openai:completion, openai:<model name>, or path to custom API caller module')
        .option('-c, --config <paths...>', 'Path to configuration file. Automatically loads promptfooconfig.js/json/yaml')
        .option(
    // TODO(ian): Remove `vars` for v1
    '-v, --vars, -t, --tests <path>', 'Path to CSV with test cases', defaultConfig?.commandLineOptions?.vars)
        .option('-a, --assertions <path>', 'Path to assertions file')
        .option('--model-outputs <path>', 'Path to JSON containing list of LLM output strings')
        .option('-t, --tests <path>', 'Path to CSV with test cases')
        .option('-o, --output <paths...>', 'Path to output file (csv, txt, json, yaml, yml, html), default is no output file')
        .option('-j, --max-concurrency <number>', 'Maximum number of concurrent API calls', defaultConfig.evaluateOptions?.maxConcurrency
        ? String(defaultConfig.evaluateOptions.maxConcurrency)
        : `${evaluator_1.DEFAULT_MAX_CONCURRENCY}`)
        .option('--repeat <number>', 'Number of times to run each test', defaultConfig.evaluateOptions?.repeat ? String(defaultConfig.evaluateOptions.repeat) : '1')
        .option('--delay <number>', 'Delay between each test (in milliseconds)', defaultConfig.evaluateOptions?.delay ? String(defaultConfig.evaluateOptions.delay) : '0')
        .option('--table-cell-max-length <number>', 'Truncate console table cells to this length', '250')
        .option('--suggest-prompts <number>', 'Generate N new prompts and append them to the prompt list')
        .option('--prompt-prefix <path>', 'This prefix is prepended to every prompt', defaultConfig.defaultTest?.options?.prefix)
        .option('--prompt-suffix <path>', 'This suffix is append to every prompt', defaultConfig.defaultTest?.options?.suffix)
        .option('--no-write', 'Do not write results to promptfoo directory', defaultConfig?.commandLineOptions?.write)
        .option('--no-cache', 'Do not read or write results to disk cache', 
    // TODO(ian): Remove commandLineOptions.cache in v1
    defaultConfig?.commandLineOptions?.cache ?? defaultConfig?.evaluateOptions?.cache)
        .option('--no-progress-bar', 'Do not show progress bar')
        .option('--table', 'Output table in CLI', defaultConfig?.commandLineOptions?.table ?? true)
        .option('--no-table', 'Do not output table in CLI', defaultConfig?.commandLineOptions?.table)
        .option('--share', 'Create a shareable URL', defaultConfig?.commandLineOptions?.share)
        .option('--grader <provider>', 'Model that will grade outputs', defaultConfig?.commandLineOptions?.grader)
        .option('--verbose', 'Show debug logs', defaultConfig?.commandLineOptions?.verbose)
        .option('-w, --watch', 'Watch for changes in config and re-run')
        .option('--env-file <path>', 'Path to .env file')
        .option('--interactive-providers', 'Run providers interactively, one at a time', defaultConfig?.evaluateOptions?.interactiveProviders)
        .option('-n, --filter-first-n <number>', 'Only run the first N tests')
        .option('--filter-pattern <pattern>', 'Only run tests whose description matches the regular expression pattern')
        .option('--filter-failing <path>', 'Path to json output file')
        .option('--var <key=value>', 'Set a variable in key=value format', (value, previous = {}) => {
        const [key, val] = value.split('=');
        if (!key || val === undefined) {
            throw new Error('--var must be specified in key=value format.');
        }
        previous[key] = val;
        return previous;
    }, {})
        .action(async (cmdObj) => {
        (0, util_1.setupEnv)(cmdObj.envFile);
        let config = undefined;
        let testSuite = undefined;
        let basePath = undefined;
        const runEvaluation = async (initialization) => {
            const startTime = Date.now();
            telemetry_1.default.record('command_used', {
                name: 'eval - started',
                watch: Boolean(cmdObj.watch),
            });
            await telemetry_1.default.send();
            // Misc settings
            if (cmdObj.verbose) {
                (0, logger_1.setLogLevel)('debug');
            }
            const iterations = parseInt(cmdObj.repeat || '', 10);
            const repeat = !isNaN(iterations) && iterations > 0 ? iterations : 1;
            if (!cmdObj.cache || repeat > 1) {
                logger_1.default.info('Cache is disabled.');
                (0, cache_1.disableCache)();
            }
            ({ config, testSuite, basePath } = await resolveConfigs(cmdObj, defaultConfig));
            cliState_1.default.basePath = basePath;
            let maxConcurrency = parseInt(cmdObj.maxConcurrency || '', 10);
            const delay = parseInt(cmdObj.delay || '', 0);
            if (delay > 0) {
                maxConcurrency = 1;
                logger_1.default.info(`Running at concurrency=1 because ${delay}ms delay was requested between API calls`);
            }
            testSuite.tests = await (0, filterTests_1.filterTests)(testSuite, {
                firstN: cmdObj.filterFirstN,
                pattern: cmdObj.filterPattern,
                failing: cmdObj.filterFailing,
            });
            const options = {
                showProgressBar: (0, logger_1.getLogLevel)() === 'debug' ? false : cmdObj.progressBar,
                maxConcurrency: !isNaN(maxConcurrency) && maxConcurrency > 0 ? maxConcurrency : undefined,
                repeat,
                delay: !isNaN(delay) && delay > 0 ? delay : undefined,
                interactiveProviders: cmdObj.interactiveProviders,
                ...evaluateOptions,
            };
            if (cmdObj.grader) {
                testSuite.defaultTest = testSuite.defaultTest || {};
                testSuite.defaultTest.options = testSuite.defaultTest.options || {};
                testSuite.defaultTest.options.provider = await (0, providers_1.loadApiProvider)(cmdObj.grader);
            }
            if (cmdObj.var) {
                testSuite.defaultTest = testSuite.defaultTest || {};
                testSuite.defaultTest.vars = { ...testSuite.defaultTest.vars, ...cmdObj.var };
            }
            if (cmdObj.generateSuggestions) {
                options.generateSuggestions = true;
            }
            const summary = await (0, evaluator_1.evaluate)(testSuite, {
                ...options,
                eventSource: 'cli',
            });
            const shareableUrl = cmdObj.share && config.sharing ? await (0, share_1.createShareableUrl)(summary, config) : null;
            if (cmdObj.table && (0, logger_1.getLogLevel)() !== 'debug') {
                // Output CLI table
                const table = (0, table_1.generateTable)(summary, parseInt(cmdObj.tableCellMaxLength || '', 10));
                logger_1.default.info('\n' + table.toString());
                if (summary.table.body.length > 25) {
                    const rowsLeft = summary.table.body.length - 25;
                    logger_1.default.info(`... ${rowsLeft} more row${rowsLeft === 1 ? '' : 's'} not shown ...\n`);
                }
            }
            else if (summary.stats.failures !== 0) {
                logger_1.default.debug(`At least one evaluation failure occurred. This might be caused by the underlying call to the provider, or a test failure. Context: \n${JSON.stringify(summary.results)}`);
            }
            await (0, util_1.migrateResultsFromFileSystemToDatabase)();
            let evalId = null;
            if (cmdObj.write) {
                evalId = await (0, util_1.writeResultsToDatabase)(summary, config);
            }
            const { outputPath } = config;
            if (outputPath) {
                // Write output to file
                if (typeof outputPath === 'string') {
                    await (0, util_1.writeOutput)(outputPath, evalId, summary, config, shareableUrl);
                }
                else if (Array.isArray(outputPath)) {
                    await (0, util_1.writeMultipleOutputs)(outputPath, evalId, summary, config, shareableUrl);
                }
                logger_1.default.info(chalk_1.default.yellow(`Writing output to ${outputPath}`));
            }
            telemetry_1.default.maybeShowNotice();
            (0, util_1.printBorder)();
            if (!cmdObj.write) {
                logger_1.default.info(`${chalk_1.default.green('✔')} Evaluation complete`);
            }
            else {
                if (shareableUrl) {
                    logger_1.default.info(`${chalk_1.default.green('✔')} Evaluation complete: ${shareableUrl}`);
                }
                else {
                    logger_1.default.info(`${chalk_1.default.green('✔')} Evaluation complete.\n`);
                    logger_1.default.info(`» Run ${chalk_1.default.greenBright.bold('promptfoo view')} to use the local web viewer`);
                    logger_1.default.info(`» Run ${chalk_1.default.greenBright.bold('promptfoo share')} to create a shareable URL`);
                    logger_1.default.info(`» This project needs your feedback. What's one thing we can improve? ${chalk_1.default.greenBright.bold('https://forms.gle/YFLgTe1dKJKNSCsU7')}`);
                }
            }
            (0, util_1.printBorder)();
            logger_1.default.info(chalk_1.default.green.bold(`Successes: ${summary.stats.successes}`));
            logger_1.default.info(chalk_1.default.red.bold(`Failures: ${summary.stats.failures}`));
            logger_1.default.info(`Token usage: Total ${summary.stats.tokenUsage.total}, Prompt ${summary.stats.tokenUsage.prompt}, Completion ${summary.stats.tokenUsage.completion}, Cached ${summary.stats.tokenUsage.cached}`);
            telemetry_1.default.record('command_used', {
                name: 'eval',
                watch: Boolean(cmdObj.watch),
                duration: Math.round((Date.now() - startTime) / 1000),
            });
            await telemetry_1.default.send();
            if (cmdObj.watch) {
                if (initialization) {
                    const configPaths = (cmdObj.config || [defaultConfigPath]).filter(Boolean);
                    if (!configPaths.length) {
                        logger_1.default.error('Could not locate config file(s) to watch');
                        process.exit(1);
                    }
                    const basePath = path_1.default.dirname(configPaths[0]);
                    const promptPaths = Array.isArray(config.prompts)
                        ? config.prompts
                            .map((p) => {
                            if (typeof p === 'string' && p.startsWith('file://')) {
                                return path_1.default.resolve(basePath, p.slice('file://'.length));
                            }
                            else if (typeof p === 'object' && p.id && p.id.startsWith('file://')) {
                                return path_1.default.resolve(basePath, p.id.slice('file://'.length));
                            }
                            return null;
                        })
                            .filter(Boolean)
                        : [];
                    const providerPaths = Array.isArray(config.providers)
                        ? config.providers
                            .map((p) => typeof p === 'string' && p.startsWith('file://')
                            ? path_1.default.resolve(basePath, p.slice('file://'.length))
                            : null)
                            .filter(Boolean)
                        : [];
                    const varPaths = Array.isArray(config.tests)
                        ? config.tests
                            .flatMap((t) => {
                            if (typeof t === 'string' && t.startsWith('file://')) {
                                return path_1.default.resolve(basePath, t.slice('file://'.length));
                            }
                            else if (typeof t !== 'string' && t.vars) {
                                return Object.values(t.vars).flatMap((v) => {
                                    if (typeof v === 'string' && v.startsWith('file://')) {
                                        return path_1.default.resolve(basePath, v.slice('file://'.length));
                                    }
                                    return [];
                                });
                            }
                            return [];
                        })
                            .filter(Boolean)
                        : [];
                    const watchPaths = Array.from(new Set([...configPaths, ...promptPaths, ...providerPaths, ...varPaths]));
                    const watcher = chokidar_1.default.watch(watchPaths, { ignored: /^\./, persistent: true });
                    watcher
                        .on('change', async (path) => {
                        (0, util_1.printBorder)();
                        logger_1.default.info(`File change detected: ${path}`);
                        (0, util_1.printBorder)();
                        await runEvaluation();
                    })
                        .on('error', (error) => logger_1.default.error(`Watcher error: ${error}`))
                        .on('ready', () => watchPaths.forEach((watchPath) => logger_1.default.info(`Watching for file changes on ${watchPath} ...`)));
                }
            }
            else {
                logger_1.default.info('Done.');
                if (summary.stats.failures > 0) {
                    const exitCode = Number(process.env.PROMPTFOO_FAILED_TEST_EXIT_CODE);
                    process.exit(isNaN(exitCode) ? 100 : exitCode);
                }
            }
        };
        await runEvaluation(true /* initialization */);
    });
    (0, list_1.listCommand)(program);
    (0, show_1.showCommand)(program);
    (0, delete_1.deleteCommand)(program);
    (0, import_1.importCommand)(program);
    (0, export_1.exportCommand)(program);
    (0, config_1.configCommand)(program);
    program.parse(process.argv);
    if (!process.argv.slice(2).length) {
        program.outputHelp();
    }
}
if (require.main === module) {
    main();
}
//# sourceMappingURL=main.js.map