"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.renderVarsInObject = exports.safeJsonStringify = exports.resultIsForTestCase = exports.varsMatch = exports.providerToIdentifier = exports.getStandaloneEvals = exports.setupEnv = exports.transformOutput = exports.printBorder = exports.getNunjucksEngine = exports.readFilters = exports.deleteEval = exports.getEvalsWithPredicate = exports.getEvalFromId = exports.getEvals = exports.getDatasetFromHash = exports.getPromptFromHash = exports.getTestCasesWithPredicate = exports.getTestCases = exports.getPromptsWithPredicate = exports.getPrompts = exports.sha256 = exports.getPromptsForTestCasesHash = exports.getPromptsForTestCases = exports.readLatestResults = exports.updateResult = exports.readResult_fileSystem = exports.readResult = exports.dateToFilename = exports.filenameToDate = exports.cleanupOldFileResults = exports.migrateResultsFromFileSystemToDatabase = exports.listPreviousResults_fileSystem = exports.listPreviousResultFilenames_fileSystem = exports.listPreviousResults = exports.writeResultsToDatabase = exports.getLatestResultsPath = exports.setConfigDirectoryPath = exports.getConfigDirectoryPath = exports.readOutput = exports.writeOutput = exports.writeMultipleOutputs = exports.readConfigs = exports.readConfig = exports.dereferenceConfig = exports.maybeReadConfig = void 0;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const os = __importStar(require("os"));
const crypto_1 = require("crypto");
const dotenv_1 = __importDefault(require("dotenv"));
const json_schema_ref_parser_1 = __importDefault(require("@apidevtools/json-schema-ref-parser"));
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
const nunjucks_1 = __importDefault(require("nunjucks"));
const js_yaml_1 = __importDefault(require("js-yaml"));
const fast_deep_equal_1 = __importDefault(require("fast-deep-equal"));
const sync_1 = require("csv-stringify/sync");
const glob_1 = require("glob");
const drizzle_orm_1 = require("drizzle-orm");
const cliState_1 = __importDefault(require("./cliState"));
const logger_1 = __importDefault(require("./logger"));
const esm_1 = require("./esm");
const testCases_1 = require("./testCases");
const database_1 = require("./database");
const migrate_1 = require("./migrate");
const wrapper_1 = require("./python/wrapper");
const types_1 = require("./types");
const googleSheets_1 = require("./googleSheets");
const DEFAULT_QUERY_LIMIT = 100;
async function maybeReadConfig(configPath) {
    if (!fs.existsSync(configPath)) {
        return undefined;
    }
    return readConfig(configPath);
}
exports.maybeReadConfig = maybeReadConfig;
async function dereferenceConfig(rawConfig) {
    if (process.env.PROMPTFOO_DISABLE_REF_PARSER) {
        return rawConfig;
    }
    // Track and delete tools[i].function for each tool, preserving the rest of the properties
    // https://github.com/promptfoo/promptfoo/issues/364
    // Remove parameters from functions and tools to prevent dereferencing
    const extractFunctionParameters = (functions) => {
        return functions.map((func) => {
            const { parameters } = func;
            delete func.parameters;
            return { parameters };
        });
    };
    const extractToolParameters = (tools) => {
        return tools.map((tool) => {
            const { parameters } = tool.function || {};
            if (tool.function?.parameters) {
                delete tool.function.parameters;
            }
            return { parameters };
        });
    };
    // Restore parameters to functions and tools after dereferencing
    const restoreFunctionParameters = (functions, parametersList) => {
        functions.forEach((func, index) => {
            if (parametersList[index]?.parameters) {
                func.parameters = parametersList[index].parameters;
            }
        });
    };
    const restoreToolParameters = (tools, parametersList) => {
        tools.forEach((tool, index) => {
            if (parametersList[index]?.parameters) {
                tool.function = tool.function || {};
                tool.function.parameters = parametersList[index].parameters;
            }
        });
    };
    const functionsParametersList = [];
    const toolsParametersList = [];
    if (Array.isArray(rawConfig.providers)) {
        rawConfig.providers.forEach((provider, providerIndex) => {
            if (typeof provider === 'string')
                return;
            if (typeof provider === 'function')
                return;
            if (!provider.config) {
                // Handle when provider is a map
                provider = Object.values(provider)[0];
            }
            if (provider.config?.functions) {
                functionsParametersList[providerIndex] = extractFunctionParameters(provider.config.functions);
            }
            if (provider.config?.tools) {
                toolsParametersList[providerIndex] = extractToolParameters(provider.config.tools);
            }
        });
    }
    // Dereference JSON
    const config = (await json_schema_ref_parser_1.default.dereference(rawConfig));
    // Restore functions and tools parameters
    if (Array.isArray(config.providers)) {
        config.providers.forEach((provider, index) => {
            if (typeof provider === 'string')
                return;
            if (typeof provider === 'function')
                return;
            if (!provider.config) {
                // Handle when provider is a map
                provider = Object.values(provider)[0];
            }
            if (functionsParametersList[index]) {
                provider.config.functions = provider.config.functions || [];
                restoreFunctionParameters(provider.config.functions, functionsParametersList[index]);
            }
            if (toolsParametersList[index]) {
                provider.config.tools = provider.config.tools || [];
                restoreToolParameters(provider.config.tools, toolsParametersList[index]);
            }
        });
    }
    return config;
}
exports.dereferenceConfig = dereferenceConfig;
async function readConfig(configPath) {
    const ext = path.parse(configPath).ext;
    switch (ext) {
        case '.json':
        case '.yaml':
        case '.yml':
            const rawConfig = js_yaml_1.default.load(fs.readFileSync(configPath, 'utf-8'));
            return dereferenceConfig(rawConfig);
        case '.js':
        case '.cjs':
        case '.mjs':
            return (await (0, esm_1.importModule)(configPath));
        default:
            throw new Error(`Unsupported configuration file format: ${ext}`);
    }
}
exports.readConfig = readConfig;
/**
 * Reads multiple configuration files and combines them into a single UnifiedConfig.
 *
 * @param {string[]} configPaths - An array of paths to configuration files. Supports glob patterns.
 * @returns {Promise<UnifiedConfig>} A promise that resolves to a unified configuration object.
 */
async function readConfigs(configPaths) {
    const configs = [];
    for (const configPath of configPaths) {
        const globPaths = (0, glob_1.globSync)(configPath, {
            windowsPathsNoEscape: true,
        });
        if (globPaths.length === 0) {
            throw new Error(`No configuration file found at ${configPath}`);
        }
        for (const globPath of globPaths) {
            const config = await readConfig(globPath);
            configs.push(config);
        }
    }
    const providers = [];
    const seenProviders = new Set();
    configs.forEach((config) => {
        (0, tiny_invariant_1.default)(typeof config.providers !== 'function', 'Providers cannot be a function for multiple configs');
        if (typeof config.providers === 'string') {
            if (!seenProviders.has(config.providers)) {
                providers.push(config.providers);
                seenProviders.add(config.providers);
            }
        }
        else if (Array.isArray(config.providers)) {
            config.providers.forEach((provider) => {
                if (!seenProviders.has(JSON.stringify(provider))) {
                    providers.push(provider);
                    seenProviders.add(JSON.stringify(provider));
                }
            });
        }
    });
    const tests = [];
    for (const config of configs) {
        if (typeof config.tests === 'string') {
            const newTests = await (0, testCases_1.readTests)(config.tests, path.dirname(configPaths[0]));
            tests.push(...newTests);
        }
        else if (Array.isArray(config.tests)) {
            tests.push(...config.tests);
        }
    }
    const configsAreStringOrArray = configs.every((config) => typeof config.prompts === 'string' || Array.isArray(config.prompts));
    const configsAreObjects = configs.every((config) => typeof config.prompts === 'object');
    let prompts = configsAreStringOrArray ? [] : {};
    const makeAbsolute = (configPath, relativePath) => {
        if (typeof relativePath === 'string') {
            if (relativePath.startsWith('file://')) {
                relativePath =
                    'file://' + path.resolve(path.dirname(configPath), relativePath.slice('file://'.length));
            }
            return relativePath;
        }
        else if (typeof relativePath === 'object' && relativePath.id) {
            if (relativePath.id.startsWith('file://')) {
                relativePath.id =
                    'file://' +
                        path.resolve(path.dirname(configPath), relativePath.id.slice('file://'.length));
            }
            return relativePath;
        }
        else {
            throw new Error('Invalid prompt object');
        }
    };
    const seenPrompts = new Set();
    const addSeenPrompt = (prompt) => {
        if (typeof prompt === 'string') {
            seenPrompts.add(prompt);
        }
        else if (typeof prompt === 'object' && prompt.id) {
            seenPrompts.add(prompt);
        }
        else {
            throw new Error('Invalid prompt object');
        }
    };
    configs.forEach((config, idx) => {
        if (typeof config.prompts === 'string') {
            (0, tiny_invariant_1.default)(Array.isArray(prompts), 'Cannot mix string and map-type prompts');
            const absolutePrompt = makeAbsolute(configPaths[idx], config.prompts);
            addSeenPrompt(absolutePrompt);
        }
        else if (Array.isArray(config.prompts)) {
            (0, tiny_invariant_1.default)(Array.isArray(prompts), 'Cannot mix configs with map and array-type prompts');
            config.prompts
                .map((prompt) => makeAbsolute(configPaths[idx], prompt))
                .forEach((prompt) => addSeenPrompt(prompt));
        }
        else {
            // Object format such as { 'prompts/prompt1.txt': 'foo', 'prompts/prompt2.txt': 'bar' }
            (0, tiny_invariant_1.default)(typeof prompts === 'object', 'Cannot mix configs with map and array-type prompts');
            prompts = { ...prompts, ...config.prompts };
        }
    });
    if (Array.isArray(prompts)) {
        prompts.push(...Array.from(seenPrompts));
    }
    // Combine all configs into a single UnifiedConfig
    const combinedConfig = {
        description: configs.map((config) => config.description).join(', '),
        providers,
        prompts,
        tests,
        scenarios: configs.flatMap((config) => config.scenarios || []),
        defaultTest: configs.reduce((prev, curr) => {
            return {
                ...prev,
                ...curr.defaultTest,
                vars: { ...prev?.vars, ...curr.defaultTest?.vars },
                assert: [...(prev?.assert || []), ...(curr.defaultTest?.assert || [])],
                options: { ...prev?.options, ...curr.defaultTest?.options },
            };
        }, {}),
        nunjucksFilters: configs.reduce((prev, curr) => ({ ...prev, ...curr.nunjucksFilters }), {}),
        env: configs.reduce((prev, curr) => ({ ...prev, ...curr.env }), {}),
        evaluateOptions: configs.reduce((prev, curr) => ({ ...prev, ...curr.evaluateOptions }), {}),
        commandLineOptions: configs.reduce((prev, curr) => ({ ...prev, ...curr.commandLineOptions }), {}),
        metadata: configs.reduce((prev, curr) => ({ ...prev, ...curr.metadata }), {}),
        sharing: !configs.some((config) => config.sharing === false),
    };
    return combinedConfig;
}
exports.readConfigs = readConfigs;
async function writeMultipleOutputs(outputPaths, evalId, results, config, shareableUrl) {
    await Promise.all(outputPaths.map((outputPath) => writeOutput(outputPath, evalId, results, config, shareableUrl)));
}
exports.writeMultipleOutputs = writeMultipleOutputs;
async function writeOutput(outputPath, evalId, results, config, shareableUrl) {
    const outputExtension = outputPath.split('.').pop()?.toLowerCase();
    const outputToSimpleString = (output) => {
        const passFailText = output.pass ? '[PASS]' : '[FAIL]';
        const namedScoresText = Object.entries(output.namedScores)
            .map(([name, value]) => `${name}: ${value.toFixed(2)}`)
            .join(', ');
        const scoreText = namedScoresText.length > 0
            ? `(${output.score.toFixed(2)}, ${namedScoresText})`
            : `(${output.score.toFixed(2)})`;
        const gradingResultText = output.gradingResult
            ? `${output.pass ? 'Pass' : 'Fail'} Reason: ${output.gradingResult.reason}`
            : '';
        return `${passFailText} ${scoreText}

${output.text}

${gradingResultText}`.trim();
    };
    if (outputPath.match(/^https:\/\/docs\.google\.com\/spreadsheets\//)) {
        const rows = results.table.body.map((row) => {
            const csvRow = {};
            results.table.head.vars.forEach((varName, index) => {
                csvRow[varName] = row.vars[index];
            });
            results.table.head.prompts.forEach((prompt, index) => {
                csvRow[prompt.label] = outputToSimpleString(row.outputs[index]);
            });
            return csvRow;
        });
        await (0, googleSheets_1.writeCsvToGoogleSheet)(rows, outputPath);
    }
    else {
        // Ensure the directory exists
        const outputDir = path.dirname(outputPath);
        if (!fs.existsSync(outputDir)) {
            fs.mkdirSync(outputDir, { recursive: true });
        }
        if (outputExtension === 'csv') {
            const csvOutput = (0, sync_1.stringify)([
                [
                    ...results.table.head.vars,
                    ...results.table.head.prompts.map((prompt) => `[${prompt.provider}] ${prompt.label}`),
                ],
                ...results.table.body.map((row) => [...row.vars, ...row.outputs.map(outputToSimpleString)]),
            ]);
            fs.writeFileSync(outputPath, csvOutput);
        }
        else if (outputExtension === 'json') {
            fs.writeFileSync(outputPath, JSON.stringify({ evalId, results, config, shareableUrl }, null, 2));
        }
        else if (outputExtension === 'yaml' ||
            outputExtension === 'yml' ||
            outputExtension === 'txt') {
            fs.writeFileSync(outputPath, js_yaml_1.default.dump({ results, config, shareableUrl }));
        }
        else if (outputExtension === 'html') {
            const template = fs.readFileSync(`${(0, esm_1.getDirectory)()}/tableOutput.html`, 'utf-8');
            const table = [
                [
                    ...results.table.head.vars,
                    ...results.table.head.prompts.map((prompt) => `[${prompt.provider}] ${prompt.label}`),
                ],
                ...results.table.body.map((row) => [...row.vars, ...row.outputs.map(outputToSimpleString)]),
            ];
            const htmlOutput = getNunjucksEngine().renderString(template, {
                config,
                table,
                results: results.results,
            });
            fs.writeFileSync(outputPath, htmlOutput);
        }
        else {
            throw new Error(`Unsupported output file format ${outputExtension}, please use csv, txt, json, yaml, yml, html.`);
        }
    }
}
exports.writeOutput = writeOutput;
async function readOutput(outputPath) {
    const ext = path.parse(outputPath).ext.slice(1);
    switch (ext) {
        case 'json':
            return JSON.parse(fs.readFileSync(outputPath, 'utf-8'));
        default:
            throw new Error(`Unsupported output file format: ${ext} currently only supports json`);
    }
}
exports.readOutput = readOutput;
let configDirectoryPath = process.env.PROMPTFOO_CONFIG_DIR;
function getConfigDirectoryPath(createIfNotExists = false) {
    const p = configDirectoryPath || path.join(os.homedir(), '.promptfoo');
    if (createIfNotExists && !fs.existsSync(p)) {
        fs.mkdirSync(p, { recursive: true });
    }
    return p;
}
exports.getConfigDirectoryPath = getConfigDirectoryPath;
function setConfigDirectoryPath(newPath) {
    configDirectoryPath = newPath;
}
exports.setConfigDirectoryPath = setConfigDirectoryPath;
/**
 * TODO(ian): Remove this
 * @deprecated Use readLatestResults directly instead.
 */
function getLatestResultsPath() {
    return path.join(getConfigDirectoryPath(), 'output', 'latest.json');
}
exports.getLatestResultsPath = getLatestResultsPath;
async function writeResultsToDatabase(results, config, createdAt) {
    createdAt = createdAt || (results.timestamp ? new Date(results.timestamp) : new Date());
    const evalId = `eval-${createdAt.toISOString().slice(0, 19)}`;
    const db = (0, database_1.getDb)();
    const promises = [];
    promises.push(db
        .insert(database_1.evals)
        .values({
        id: evalId,
        createdAt: createdAt.getTime(),
        description: config.description,
        config,
        results,
    })
        .onConflictDoNothing()
        .run());
    logger_1.default.debug(`Inserting eval ${evalId}`);
    // Record prompt relation
    for (const prompt of results.table.head.prompts) {
        const label = prompt.label || prompt.display || prompt.raw;
        const promptId = sha256(label);
        promises.push(db
            .insert(database_1.prompts)
            .values({
            id: promptId,
            prompt: label,
        })
            .onConflictDoNothing()
            .run());
        promises.push(db
            .insert(database_1.evalsToPrompts)
            .values({
            evalId,
            promptId,
        })
            .onConflictDoNothing()
            .run());
        logger_1.default.debug(`Inserting prompt ${promptId}`);
    }
    // Record dataset relation
    const datasetId = sha256(JSON.stringify(config.tests || []));
    promises.push(db
        .insert(database_1.datasets)
        .values({
        id: datasetId,
        tests: config.tests,
    })
        .onConflictDoNothing()
        .run());
    promises.push(db
        .insert(database_1.evalsToDatasets)
        .values({
        evalId,
        datasetId,
    })
        .onConflictDoNothing()
        .run());
    logger_1.default.debug(`Inserting dataset ${datasetId}`);
    logger_1.default.debug(`Awaiting ${promises.length} promises to database...`);
    await Promise.all(promises);
    // "touch" db signal path
    const filePath = (0, database_1.getDbSignalPath)();
    try {
        const now = new Date();
        fs.utimesSync(filePath, now, now);
    }
    catch (err) {
        fs.closeSync(fs.openSync(filePath, 'w'));
    }
    return evalId;
}
exports.writeResultsToDatabase = writeResultsToDatabase;
/**
 *
 * @returns Last n evals in descending order.
 */
function listPreviousResults(limit = DEFAULT_QUERY_LIMIT, filterDescription) {
    const db = (0, database_1.getDb)();
    let results = db
        .select({
        name: database_1.evals.id,
        description: database_1.evals.description,
    })
        .from(database_1.evals)
        .orderBy((0, drizzle_orm_1.desc)(database_1.evals.createdAt))
        .limit(limit)
        .all();
    if (filterDescription) {
        const regex = new RegExp(filterDescription, 'i');
        results = results.filter((result) => regex.test(result.description || ''));
    }
    return results.map((result) => ({
        evalId: result.name,
        description: result.description,
    }));
}
exports.listPreviousResults = listPreviousResults;
/**
 * @deprecated Used only for migration to sqlite
 */
function listPreviousResultFilenames_fileSystem() {
    const directory = path.join(getConfigDirectoryPath(), 'output');
    if (!fs.existsSync(directory)) {
        return [];
    }
    const files = fs.readdirSync(directory);
    const resultsFiles = files.filter((file) => file.startsWith('eval-') && file.endsWith('.json'));
    return resultsFiles.sort((a, b) => {
        const statA = fs.statSync(path.join(directory, a));
        const statB = fs.statSync(path.join(directory, b));
        return statA.birthtime.getTime() - statB.birthtime.getTime(); // sort in ascending order
    });
}
exports.listPreviousResultFilenames_fileSystem = listPreviousResultFilenames_fileSystem;
const resultsCache = {};
/**
 * @deprecated Used only for migration to sqlite
 */
function listPreviousResults_fileSystem() {
    const directory = path.join(getConfigDirectoryPath(), 'output');
    if (!fs.existsSync(directory)) {
        return [];
    }
    const sortedFiles = listPreviousResultFilenames_fileSystem();
    return sortedFiles.map((fileName) => {
        if (!resultsCache[fileName]) {
            try {
                const fileContents = fs.readFileSync(path.join(directory, fileName), 'utf8');
                const data = js_yaml_1.default.load(fileContents);
                resultsCache[fileName] = data;
            }
            catch (error) {
                logger_1.default.warn(`Failed to read results from ${fileName}:\n${error}`);
            }
        }
        return {
            fileName,
            description: resultsCache[fileName]?.config.description,
        };
    });
}
exports.listPreviousResults_fileSystem = listPreviousResults_fileSystem;
let attemptedMigration = false;
async function migrateResultsFromFileSystemToDatabase() {
    if (attemptedMigration) {
        // TODO(ian): Record this bit in the database.
        return;
    }
    // First run db migrations
    logger_1.default.debug('Running db migrations...');
    await (0, migrate_1.runDbMigrations)();
    const fileNames = listPreviousResultFilenames_fileSystem();
    if (fileNames.length === 0) {
        return;
    }
    logger_1.default.info(`🔁 Migrating ${fileNames.length} flat files to local database.`);
    logger_1.default.info('This is a one-time operation and may take a minute...');
    attemptedMigration = true;
    const outputDir = path.join(getConfigDirectoryPath(true /* createIfNotExists */), 'output');
    const backupDir = `${outputDir}-backup-${new Date()
        .toISOString()
        .slice(0, 10)
        .replace(/-/g, '')}`;
    try {
        fs.cpSync(outputDir, backupDir, { recursive: true });
        logger_1.default.info(`Backup of output directory created at ${backupDir}`);
    }
    catch (backupError) {
        logger_1.default.error(`Failed to create backup of output directory: ${backupError}`);
        return;
    }
    logger_1.default.info('Moving files into database...');
    const migrationPromises = fileNames.map(async (fileName) => {
        const fileData = readResult_fileSystem(fileName);
        if (fileData) {
            await writeResultsToDatabase(fileData.result.results, fileData.result.config, filenameToDate(fileName));
            logger_1.default.debug(`Migrated ${fileName} to database.`);
            try {
                fs.unlinkSync(path.join(outputDir, fileName));
            }
            catch (err) {
                logger_1.default.warn(`Failed to delete ${fileName} after migration: ${err}`);
            }
        }
        else {
            logger_1.default.warn(`Failed to migrate result ${fileName} due to read error.`);
        }
    });
    await Promise.all(migrationPromises);
    try {
        fs.unlinkSync(getLatestResultsPath());
    }
    catch (err) {
        logger_1.default.warn(`Failed to delete latest.json: ${err}`);
    }
    logger_1.default.info('Migration complete. Please restart your web server if it is running.');
}
exports.migrateResultsFromFileSystemToDatabase = migrateResultsFromFileSystemToDatabase;
const RESULT_HISTORY_LENGTH = parseInt(process.env.RESULT_HISTORY_LENGTH || '', 10) || DEFAULT_QUERY_LIMIT;
function cleanupOldFileResults(remaining = RESULT_HISTORY_LENGTH) {
    const sortedFilenames = listPreviousResultFilenames_fileSystem();
    for (let i = 0; i < sortedFilenames.length - remaining; i++) {
        fs.unlinkSync(path.join(getConfigDirectoryPath(), 'output', sortedFilenames[i]));
    }
}
exports.cleanupOldFileResults = cleanupOldFileResults;
function filenameToDate(filename) {
    const dateString = filename.slice('eval-'.length, filename.length - '.json'.length);
    // Replace hyphens with colons where necessary (Windows compatibility).
    const dateParts = dateString.split('T');
    const timePart = dateParts[1].replace(/-/g, ':');
    const formattedDateString = `${dateParts[0]}T${timePart}`;
    const date = new Date(formattedDateString);
    return date;
    /*
    return date.toLocaleDateString('en-US', {
      year: 'numeric',
      month: 'long',
      day: 'numeric',
      hour: '2-digit',
      minute: '2-digit',
      second: '2-digit',
      timeZoneName: 'short',
    });
    */
}
exports.filenameToDate = filenameToDate;
function dateToFilename(date) {
    return `eval-${date.toISOString().replace(/:/g, '-')}.json`;
}
exports.dateToFilename = dateToFilename;
async function readResult(id) {
    const db = (0, database_1.getDb)();
    try {
        const evalResult = await db
            .select({
            id: database_1.evals.id,
            createdAt: database_1.evals.createdAt,
            results: database_1.evals.results,
            config: database_1.evals.config,
        })
            .from(database_1.evals)
            .where((0, drizzle_orm_1.eq)(database_1.evals.id, id))
            .execute();
        if (evalResult.length === 0) {
            return undefined;
        }
        const { id: resultId, createdAt, results, config } = evalResult[0];
        const result = {
            version: 3,
            createdAt: new Date(createdAt).toISOString().slice(0, 10),
            results,
            config,
        };
        return {
            id: resultId,
            result,
            createdAt: new Date(createdAt),
        };
    }
    catch (err) {
        logger_1.default.error(`Failed to read result with ID ${id} from database:\n${err}`);
    }
}
exports.readResult = readResult;
/**
 * @deprecated Used only for migration to sqlite
 */
function readResult_fileSystem(name) {
    const resultsDirectory = path.join(getConfigDirectoryPath(), 'output');
    const resultsPath = path.join(resultsDirectory, name);
    try {
        const result = JSON.parse(fs.readFileSync(fs.realpathSync(resultsPath), 'utf-8'));
        const createdAt = filenameToDate(name);
        return {
            id: sha256(JSON.stringify(result.config)),
            result,
            createdAt,
        };
    }
    catch (err) {
        logger_1.default.error(`Failed to read results from ${resultsPath}:\n${err}`);
    }
}
exports.readResult_fileSystem = readResult_fileSystem;
async function updateResult(id, newConfig, newTable) {
    const db = (0, database_1.getDb)();
    try {
        // Fetch the existing eval data from the database
        const existingEval = await db
            .select({
            config: database_1.evals.config,
            results: database_1.evals.results,
        })
            .from(database_1.evals)
            .where((0, drizzle_orm_1.eq)(database_1.evals.id, id))
            .limit(1)
            .all();
        if (existingEval.length === 0) {
            logger_1.default.error(`Eval with ID ${id} not found.`);
            return;
        }
        const evalData = existingEval[0];
        if (newConfig) {
            evalData.config = newConfig;
        }
        if (newTable) {
            evalData.results.table = newTable;
        }
        await db
            .update(database_1.evals)
            .set({
            description: evalData.config.description,
            config: evalData.config,
            results: evalData.results,
        })
            .where((0, drizzle_orm_1.eq)(database_1.evals.id, id))
            .run();
        logger_1.default.info(`Updated eval with ID ${id}`);
    }
    catch (err) {
        logger_1.default.error(`Failed to update eval with ID ${id}:\n${err}`);
    }
}
exports.updateResult = updateResult;
async function readLatestResults(filterDescription) {
    const db = (0, database_1.getDb)();
    let latestResults = await db
        .select({
        id: database_1.evals.id,
        createdAt: database_1.evals.createdAt,
        description: database_1.evals.description,
        results: database_1.evals.results,
        config: database_1.evals.config,
    })
        .from(database_1.evals)
        .orderBy((0, drizzle_orm_1.desc)(database_1.evals.createdAt))
        .limit(1);
    if (filterDescription) {
        const regex = new RegExp(filterDescription, 'i');
        latestResults = latestResults.filter((result) => regex.test(result.description || ''));
    }
    if (!latestResults.length) {
        return undefined;
    }
    const latestResult = latestResults[0];
    return {
        version: 3,
        createdAt: new Date(latestResult.createdAt).toISOString(),
        results: latestResult.results,
        config: latestResult.config,
    };
}
exports.readLatestResults = readLatestResults;
function getPromptsForTestCases(testCases) {
    const testCasesJson = JSON.stringify(testCases);
    const testCasesSha256 = sha256(testCasesJson);
    return getPromptsForTestCasesHash(testCasesSha256);
}
exports.getPromptsForTestCases = getPromptsForTestCases;
function getPromptsForTestCasesHash(testCasesSha256, limit = DEFAULT_QUERY_LIMIT) {
    return getPromptsWithPredicate((result) => {
        const testsJson = JSON.stringify(result.config.tests);
        const hash = sha256(testsJson);
        return hash === testCasesSha256;
    }, limit);
}
exports.getPromptsForTestCasesHash = getPromptsForTestCasesHash;
function sha256(str) {
    return (0, crypto_1.createHash)('sha256').update(str).digest('hex');
}
exports.sha256 = sha256;
function getPrompts(limit = DEFAULT_QUERY_LIMIT) {
    return getPromptsWithPredicate(() => true, limit);
}
exports.getPrompts = getPrompts;
async function getPromptsWithPredicate(predicate, limit) {
    // TODO(ian): Make this use a proper database query
    const db = (0, database_1.getDb)();
    const evals_ = await db
        .select({
        id: database_1.evals.id,
        createdAt: database_1.evals.createdAt,
        results: database_1.evals.results,
        config: database_1.evals.config,
    })
        .from(database_1.evals)
        .limit(limit)
        .all();
    const groupedPrompts = {};
    for (const eval_ of evals_) {
        const createdAt = new Date(eval_.createdAt).toISOString();
        const resultWrapper = {
            version: 3,
            createdAt,
            results: eval_.results,
            config: eval_.config,
        };
        if (predicate(resultWrapper)) {
            for (const prompt of resultWrapper.results.table.head.prompts) {
                const promptId = sha256(prompt.raw);
                const datasetId = resultWrapper.config.tests
                    ? sha256(JSON.stringify(resultWrapper.config.tests))
                    : '-';
                if (promptId in groupedPrompts) {
                    groupedPrompts[promptId].recentEvalDate = new Date(Math.max(groupedPrompts[promptId].recentEvalDate.getTime(), new Date(createdAt).getTime()));
                    groupedPrompts[promptId].count += 1;
                    groupedPrompts[promptId].evals.push({
                        id: eval_.id,
                        datasetId,
                        metrics: prompt.metrics,
                    });
                }
                else {
                    groupedPrompts[promptId] = {
                        count: 1,
                        id: promptId,
                        prompt,
                        recentEvalDate: new Date(createdAt),
                        recentEvalId: eval_.id,
                        evals: [
                            {
                                id: eval_.id,
                                datasetId,
                                metrics: prompt.metrics,
                            },
                        ],
                    };
                }
            }
        }
    }
    return Object.values(groupedPrompts);
}
exports.getPromptsWithPredicate = getPromptsWithPredicate;
async function getTestCases(limit = DEFAULT_QUERY_LIMIT) {
    return getTestCasesWithPredicate(() => true, limit);
}
exports.getTestCases = getTestCases;
async function getTestCasesWithPredicate(predicate, limit) {
    const db = (0, database_1.getDb)();
    const evals_ = await db
        .select({
        id: database_1.evals.id,
        createdAt: database_1.evals.createdAt,
        results: database_1.evals.results,
        config: database_1.evals.config,
    })
        .from(database_1.evals)
        .limit(limit)
        .all();
    const groupedTestCases = {};
    for (const eval_ of evals_) {
        const createdAt = new Date(eval_.createdAt).toISOString();
        const resultWrapper = {
            version: 3,
            createdAt,
            results: eval_.results,
            config: eval_.config,
        };
        const testCases = resultWrapper.config.tests;
        if (testCases && predicate(resultWrapper)) {
            const evalId = eval_.id;
            const datasetId = sha256(JSON.stringify(testCases));
            if (datasetId in groupedTestCases) {
                groupedTestCases[datasetId].recentEvalDate = new Date(Math.max(groupedTestCases[datasetId].recentEvalDate.getTime(), eval_.createdAt));
                groupedTestCases[datasetId].count += 1;
                const newPrompts = resultWrapper.results.table.head.prompts.map((prompt) => ({
                    id: sha256(prompt.raw),
                    prompt,
                    evalId,
                }));
                const promptsById = {};
                for (const prompt of groupedTestCases[datasetId].prompts.concat(newPrompts)) {
                    if (!(prompt.id in promptsById)) {
                        promptsById[prompt.id] = prompt;
                    }
                }
                groupedTestCases[datasetId].prompts = Object.values(promptsById);
            }
            else {
                const newPrompts = resultWrapper.results.table.head.prompts.map((prompt) => ({
                    id: sha256(prompt.raw),
                    prompt,
                    evalId,
                }));
                const promptsById = {};
                for (const prompt of newPrompts) {
                    if (!(prompt.id in promptsById)) {
                        promptsById[prompt.id] = prompt;
                    }
                }
                groupedTestCases[datasetId] = {
                    id: datasetId,
                    count: 1,
                    testCases,
                    recentEvalDate: new Date(createdAt),
                    recentEvalId: evalId,
                    prompts: Object.values(promptsById),
                };
            }
        }
    }
    return Object.values(groupedTestCases);
}
exports.getTestCasesWithPredicate = getTestCasesWithPredicate;
async function getPromptFromHash(hash) {
    const prompts = await getPrompts();
    for (const prompt of prompts) {
        if (prompt.id.startsWith(hash)) {
            return prompt;
        }
    }
    return undefined;
}
exports.getPromptFromHash = getPromptFromHash;
async function getDatasetFromHash(hash) {
    const datasets = await getTestCases();
    for (const dataset of datasets) {
        if (dataset.id.startsWith(hash)) {
            return dataset;
        }
    }
    return undefined;
}
exports.getDatasetFromHash = getDatasetFromHash;
async function getEvals(limit = DEFAULT_QUERY_LIMIT) {
    return getEvalsWithPredicate(() => true, limit);
}
exports.getEvals = getEvals;
async function getEvalFromId(hash) {
    const evals_ = await getEvals();
    for (const eval_ of evals_) {
        if (eval_.id.startsWith(hash)) {
            return eval_;
        }
    }
    return undefined;
}
exports.getEvalFromId = getEvalFromId;
async function getEvalsWithPredicate(predicate, limit) {
    const db = (0, database_1.getDb)();
    const evals_ = await db
        .select({
        id: database_1.evals.id,
        createdAt: database_1.evals.createdAt,
        results: database_1.evals.results,
        config: database_1.evals.config,
        description: database_1.evals.description,
    })
        .from(database_1.evals)
        .orderBy((0, drizzle_orm_1.desc)(database_1.evals.createdAt))
        .limit(limit)
        .all();
    const ret = [];
    for (const eval_ of evals_) {
        const createdAt = new Date(eval_.createdAt).toISOString();
        const resultWrapper = {
            version: 3,
            createdAt: createdAt,
            results: eval_.results,
            config: eval_.config,
        };
        if (predicate(resultWrapper)) {
            const evalId = eval_.id;
            ret.push({
                id: evalId,
                date: new Date(eval_.createdAt),
                config: eval_.config,
                results: eval_.results,
                description: eval_.description || undefined,
            });
        }
    }
    return ret;
}
exports.getEvalsWithPredicate = getEvalsWithPredicate;
async function deleteEval(evalId) {
    const db = (0, database_1.getDb)();
    await db.transaction(async () => {
        // We need to clean up foreign keys first. We don't have onDelete: 'cascade' set on all these relationships.
        await db.delete(database_1.evalsToPrompts).where((0, drizzle_orm_1.eq)(database_1.evalsToPrompts.evalId, evalId)).run();
        await db.delete(database_1.evalsToDatasets).where((0, drizzle_orm_1.eq)(database_1.evalsToDatasets.evalId, evalId)).run();
        // Finally, delete the eval record
        const deletedIds = await db.delete(database_1.evals).where((0, drizzle_orm_1.eq)(database_1.evals.id, evalId)).run();
        if (deletedIds.changes === 0) {
            throw new Error(`Eval with ID ${evalId} not found`);
        }
    });
}
exports.deleteEval = deleteEval;
async function readFilters(filters) {
    const ret = {};
    const basePath = cliState_1.default.basePath || '';
    for (const [name, filterPath] of Object.entries(filters)) {
        const globPath = path.join(basePath, filterPath);
        const filePaths = (0, glob_1.globSync)(globPath, {
            windowsPathsNoEscape: true,
        });
        for (const filePath of filePaths) {
            const finalPath = path.resolve(filePath);
            ret[name] = await (0, esm_1.importModule)(finalPath);
        }
    }
    return ret;
}
exports.readFilters = readFilters;
function getNunjucksEngine(filters) {
    if (process.env.PROMPTFOO_DISABLE_TEMPLATING) {
        return {
            renderString: (template) => template,
        };
    }
    const env = nunjucks_1.default.configure({
        autoescape: false,
    });
    if (filters) {
        for (const [name, filter] of Object.entries(filters)) {
            env.addFilter(name, filter);
        }
    }
    return env;
}
exports.getNunjucksEngine = getNunjucksEngine;
function printBorder() {
    const border = '='.repeat((process.stdout.columns || 80) - 10);
    logger_1.default.info(border);
}
exports.printBorder = printBorder;
async function transformOutput(codeOrFilepath, output, context) {
    let postprocessFn;
    if (codeOrFilepath.startsWith('file://')) {
        const filePath = codeOrFilepath.slice('file://'.length);
        if (codeOrFilepath.endsWith('.js') ||
            codeOrFilepath.endsWith('.cjs') ||
            codeOrFilepath.endsWith('.mjs')) {
            const requiredModule = await (0, esm_1.importModule)(filePath);
            if (typeof requiredModule === 'function') {
                postprocessFn = requiredModule;
            }
            else if (requiredModule.default && typeof requiredModule.default === 'function') {
                postprocessFn = requiredModule.default;
            }
            else {
                throw new Error(`Transform ${filePath} must export a function or have a default export as a function`);
            }
        }
        else if (codeOrFilepath.endsWith('.py')) {
            postprocessFn = async (output, context) => {
                return (0, wrapper_1.runPython)(filePath, 'get_transform', [output, context]);
            };
        }
        else {
            throw new Error(`Unsupported transform file format: ${codeOrFilepath}`);
        }
    }
    else {
        postprocessFn = new Function('output', 'context', codeOrFilepath.includes('\n') ? codeOrFilepath : `return ${codeOrFilepath}`);
    }
    const ret = await Promise.resolve(postprocessFn(output, context));
    if (ret == null) {
        throw new Error(`Transform function did not return a value\n\n${codeOrFilepath}`);
    }
    return ret;
}
exports.transformOutput = transformOutput;
function setupEnv(envPath) {
    if (envPath) {
        logger_1.default.info(`Loading environment variables from ${envPath}`);
        dotenv_1.default.config({ path: envPath });
    }
    else {
        dotenv_1.default.config();
    }
}
exports.setupEnv = setupEnv;
function getStandaloneEvals(limit = DEFAULT_QUERY_LIMIT) {
    const db = (0, database_1.getDb)();
    const results = db
        .select({
        evalId: database_1.evals.id,
        description: database_1.evals.description,
        config: database_1.evals.config,
        results: database_1.evals.results,
        promptId: database_1.evalsToPrompts.promptId,
        datasetId: database_1.evalsToDatasets.datasetId,
    })
        .from(database_1.evals)
        .leftJoin(database_1.evalsToPrompts, (0, drizzle_orm_1.eq)(database_1.evals.id, database_1.evalsToPrompts.evalId))
        .leftJoin(database_1.evalsToDatasets, (0, drizzle_orm_1.eq)(database_1.evals.id, database_1.evalsToDatasets.evalId))
        .orderBy((0, drizzle_orm_1.desc)(database_1.evals.createdAt))
        .limit(limit)
        .all();
    const flatResults = [];
    results.forEach((result) => {
        const table = result.results.table;
        table.head.prompts.forEach((col) => {
            flatResults.push({
                evalId: result.evalId,
                promptId: result.promptId,
                datasetId: result.datasetId,
                ...col,
            });
        });
    });
    return flatResults;
}
exports.getStandaloneEvals = getStandaloneEvals;
function providerToIdentifier(provider) {
    if ((0, types_1.isApiProvider)(provider)) {
        return provider.id();
    }
    else if ((0, types_1.isProviderOptions)(provider)) {
        return provider.id;
    }
    return provider;
}
exports.providerToIdentifier = providerToIdentifier;
function varsMatch(vars1, vars2) {
    return (0, fast_deep_equal_1.default)(vars1, vars2);
}
exports.varsMatch = varsMatch;
function resultIsForTestCase(result, testCase) {
    const providersMatch = testCase.provider
        ? providerToIdentifier(testCase.provider) === providerToIdentifier(result.provider)
        : true;
    return varsMatch(testCase.vars, result.vars) && providersMatch;
}
exports.resultIsForTestCase = resultIsForTestCase;
function safeJsonStringify(value, prettyPrint = false) {
    // Prevent circular references
    const cache = new Set();
    const space = prettyPrint ? 2 : undefined;
    return JSON.stringify(value, (key, val) => {
        if (typeof val === 'object' && val !== null) {
            if (cache.has(val))
                return;
            cache.add(val);
        }
        return val;
    }, space);
}
exports.safeJsonStringify = safeJsonStringify;
function renderVarsInObject(obj, vars) {
    // Renders nunjucks template strings with context variables
    if (!vars || process.env.PROMPTFOO_DISABLE_TEMPLATING) {
        return obj;
    }
    if (typeof obj === 'string') {
        return nunjucks_1.default.renderString(obj, vars);
    }
    if (Array.isArray(obj)) {
        return obj.map((item) => renderVarsInObject(item, vars));
    }
    if (typeof obj === 'object' && obj !== null) {
        const result = {};
        for (const key in obj) {
            result[key] = renderVarsInObject(obj[key], vars);
        }
        return result;
    }
    else if (typeof obj === 'function') {
        const fn = obj;
        return renderVarsInObject(fn({ vars }));
    }
    return obj;
}
exports.renderVarsInObject = renderVarsInObject;
//# sourceMappingURL=util.js.map