const require_logger = require('./logger-B3t6CmTC.cjs');
const require_invariant = require('./invariant-CKcJAQ6M.cjs');
const require_esm = require('./esm-DRsi2FRh.cjs');
const require_pythonUtils = require('./pythonUtils-BU2j0p3m.cjs');
const require_fileExtensions = require('./fileExtensions-BDTd6vBW.cjs');
const require_fetch = require('./fetch-COTDhwC8.cjs');
const require_types = require('./types-B2TvjqC8.cjs');
let fs = require("fs");
fs = require_logger.__toESM(fs);
let path = require("path");
path = require_logger.__toESM(path);
let js_yaml = require("js-yaml");
js_yaml = require_logger.__toESM(js_yaml);
let dotenv = require("dotenv");
dotenv = require_logger.__toESM(dotenv);
let os = require("os");
os = require_logger.__toESM(os);
let dedent = require("dedent");
dedent = require_logger.__toESM(dedent);
let glob = require("glob");
let csv_stringify_sync = require("csv-stringify/sync");
let fast_deep_equal = require("fast-deep-equal");
fast_deep_equal = require_logger.__toESM(fast_deep_equal);
let fast_xml_parser = require("fast-xml-parser");
let csv_parse_sync = require("csv-parse/sync");
let nunjucks = require("nunjucks");
nunjucks = require_logger.__toESM(nunjucks);
let openai = require("openai");
openai = require_logger.__toESM(openai);

//#region src/googleSheets.ts
async function checkGoogleSheetAccess(url) {
	try {
		const response = await require_fetch.fetchWithProxy(url);
		if (response.ok) return {
			public: true,
			status: response.status
		};
		else return {
			public: false,
			status: response.status
		};
	} catch (error) {
		require_logger.logger_default.error(`Error checking sheet access: ${error}`);
		return { public: false };
	}
}
async function fetchCsvFromGoogleSheetUnauthenticated(url) {
	const { parse: parseCsv } = await import("csv-parse/sync");
	const gid = new URL(url).searchParams.get("gid");
	const response = await require_fetch.fetchWithProxy(`${url.replace(/\/edit.*$/, "/export")}?format=csv${gid ? `&gid=${gid}` : ""}`);
	if (response.status !== 200) throw new Error(`Failed to fetch CSV from Google Sheets URL: ${url}`);
	return parseCsv(await response.text(), { columns: true });
}
async function fetchCsvFromGoogleSheetAuthenticated(url) {
	const { sheets: googleSheets, auth: googleAuth } = await import("@googleapis/sheets");
	const auth = new googleAuth.GoogleAuth({ scopes: ["https://www.googleapis.com/auth/spreadsheets.readonly"] });
	const sheets = googleSheets("v4");
	const match = url.match(/\/d\/([^/]+)/);
	if (!match) throw new Error(`Invalid Google Sheets URL: ${url}`);
	const spreadsheetId = match[1];
	let range;
	const gid = Number(new URL(url).searchParams.get("gid"));
	if (gid) {
		const sheet = (await sheets.spreadsheets.get({
			spreadsheetId,
			auth
		})).data.sheets?.find((sheet$1) => sheet$1.properties?.sheetId === gid);
		if (!sheet || !sheet.properties?.title) throw new Error(`Sheet not found for gid: ${gid}`);
		range = sheet.properties.title;
	} else {
		const firstSheet = (await sheets.spreadsheets.get({
			spreadsheetId,
			auth
		})).data.sheets?.[0];
		if (!firstSheet || !firstSheet.properties?.title) throw new Error(`No sheets found in spreadsheet`);
		range = firstSheet.properties.title;
	}
	const rows = (await sheets.spreadsheets.values.get({
		spreadsheetId,
		range,
		auth
	})).data.values;
	if (!rows?.length) throw new Error(`No data found in Google Sheets URL: ${url}`);
	const headers = rows[0];
	return rows.slice(1).map((row) => {
		const csvRow = {};
		headers.forEach((header, index) => {
			csvRow[header] = row[index] ?? "";
		});
		return csvRow;
	});
}
async function fetchCsvFromGoogleSheet(url) {
	const { public: isPublic } = await checkGoogleSheetAccess(url);
	require_logger.logger_default.debug(`Google Sheets URL: ${url}, isPublic: ${isPublic}`);
	if (isPublic) return fetchCsvFromGoogleSheetUnauthenticated(url);
	return fetchCsvFromGoogleSheetAuthenticated(url);
}
async function writeCsvToGoogleSheet(rows, url) {
	const { sheets: googleSheets, auth: googleAuth } = await import("@googleapis/sheets");
	const auth = new googleAuth.GoogleAuth({ scopes: ["https://www.googleapis.com/auth/spreadsheets"] });
	const sheets = googleSheets("v4");
	const match = url.match(/\/d\/([^/]+)/);
	if (!match) throw new Error(`Invalid Google Sheets URL: ${url}`);
	const spreadsheetId = match[1];
	const headers = Object.keys(rows[0]);
	const values = [headers, ...rows.map((row) => headers.map((header) => row[header]))];
	const getColumnLetter = (col) => {
		let letter = "";
		while (col > 0) {
			col--;
			letter = String.fromCharCode(65 + col % 26) + letter;
			col = Math.floor(col / 26);
		}
		return letter;
	};
	const numRows = values.length;
	const numCols = headers.length;
	const endColumn = getColumnLetter(numCols);
	let range;
	const gid = Number(new URL(url).searchParams.get("gid"));
	if (gid) {
		const sheet = (await sheets.spreadsheets.get({
			spreadsheetId,
			auth
		})).data.sheets?.find((sheet$1) => sheet$1.properties?.sheetId === gid);
		if (!sheet || !sheet.properties?.title) throw new Error(`Sheet not found for gid: ${gid}`);
		range = `${sheet.properties.title}!A1:${endColumn}${numRows}`;
	} else {
		const newSheetTitle = `Sheet${Date.now()}`;
		await sheets.spreadsheets.batchUpdate({
			spreadsheetId,
			auth,
			requestBody: { requests: [{ addSheet: { properties: { title: newSheetTitle } } }] }
		});
		range = `${newSheetTitle}!A1:${endColumn}${numRows}`;
	}
	require_logger.logger_default.debug(`Writing CSV to Google Sheets URL: ${url} with ${values.length} rows`);
	await sheets.spreadsheets.values.update({
		spreadsheetId,
		range,
		valueInputOption: "USER_ENTERED",
		auth,
		requestBody: { values }
	});
}

//#endregion
//#region src/util/exportToFile/getHeaderForTable.ts
function getHeaderForTable(eval_) {
	const varsForHeader = /* @__PURE__ */ new Set();
	if (typeof eval_.config.defaultTest === "object" && eval_.config.defaultTest?.vars) for (const varName of Object.keys(eval_.config.defaultTest.vars || {})) varsForHeader.add(varName);
	for (const result of eval_.results) if (result.testCase?.vars) for (const varName of Object.keys(result.testCase.vars)) varsForHeader.add(varName);
	const tests = eval_.config.tests;
	let testsArray = [];
	if (Array.isArray(tests)) testsArray = tests;
	else if (tests) testsArray = [tests];
	for (const test of testsArray) {
		if (typeof test === "string") continue;
		if (test && typeof test === "object" && "path" in test) continue;
		if (test && "vars" in test) for (const varName of Object.keys(test.vars || {})) varsForHeader.add(varName);
	}
	for (const scenario of eval_.config.scenarios || []) {
		if (typeof scenario === "string") continue;
		for (const config of scenario.config || []) for (const varName of Object.keys(config.vars || {})) varsForHeader.add(varName);
		for (const test of scenario.tests || []) {
			if (typeof test === "string") continue;
			for (const varName of Object.keys(test.vars || {})) varsForHeader.add(varName);
		}
	}
	return {
		vars: [...varsForHeader].sort(),
		prompts: eval_.prompts
	};
}

//#endregion
//#region src/util/exportToFile/index.ts
function convertEvalResultToTableCell(result) {
	let resultText;
	const outputTextDisplay = typeof result.response?.output === "object" ? JSON.stringify(result.response.output) : result.response?.output || result.error || "";
	if (result.testCase.assert) if (result.success) resultText = `${outputTextDisplay || result.error || ""}`;
	else resultText = `${outputTextDisplay}`;
	else if (result.error) resultText = `${result.error}`;
	else resultText = outputTextDisplay;
	return {
		...result,
		id: result.id || `${result.testIdx}-${result.promptIdx}`,
		text: resultText || "",
		prompt: result.prompt.raw,
		provider: result.provider?.label || result.provider?.id || "unknown provider",
		pass: result.success,
		cost: result.cost || 0,
		audio: result.response?.audio ? {
			id: result.response.audio.id,
			expiresAt: result.response.audio.expiresAt,
			data: result.response.audio.data,
			transcript: result.response.audio.transcript,
			format: result.response.audio.format
		} : void 0
	};
}
function convertTestResultsToTableRow(results, varsForHeader) {
	const row = {
		description: results[0].description || void 0,
		outputs: [],
		vars: results[0].testCase.vars ? Object.values(varsForHeader).map((varName) => {
			const varValue = results[0].testCase.vars?.[varName] || "";
			if (typeof varValue === "string") return varValue;
			return JSON.stringify(varValue);
		}).flat() : [],
		test: results[0].testCase,
		testIdx: results[0].testIdx
	};
	for (const result of results) row.outputs[result.promptIdx] = convertEvalResultToTableCell(result);
	return row;
}

//#endregion
//#region src/util/functions/loadFunction.ts
const functionCache = {};
/**
* Loads a function from a JavaScript or Python file
* @param options Options for loading the function
* @returns The loaded function
*/
async function loadFunction({ filePath, functionName, defaultFunctionName = "func", basePath = require_logger.cliState_default.basePath, useCache = true }) {
	const cacheKey = `${filePath}${functionName ? `:${functionName}` : ""}`;
	if (useCache && functionCache[cacheKey]) return functionCache[cacheKey];
	const resolvedPath = basePath ? path.default.resolve(basePath, filePath) : filePath;
	if (!require_fileExtensions.isJavascriptFile(resolvedPath) && !resolvedPath.endsWith(".py")) throw new Error(`File must be a JavaScript (${require_fileExtensions.JAVASCRIPT_EXTENSIONS.join(", ")}) or Python (.py) file`);
	try {
		let func;
		if (require_fileExtensions.isJavascriptFile(resolvedPath)) {
			const module$1 = await require_esm.importModule(resolvedPath, functionName);
			let moduleFunc;
			if (functionName) moduleFunc = module$1;
			else moduleFunc = typeof module$1 === "function" ? module$1 : module$1?.default?.default || module$1?.default || module$1?.[defaultFunctionName] || module$1;
			if (typeof moduleFunc !== "function") throw new Error(functionName ? `JavaScript file must export a "${functionName}" function` : `JavaScript file must export a function (as default export or named export "${defaultFunctionName}")`);
			func = moduleFunc;
		} else {
			const result = (...args) => require_pythonUtils.runPython(resolvedPath, functionName || defaultFunctionName, args);
			func = result;
		}
		if (useCache) functionCache[cacheKey] = func;
		return func;
	} catch (err) {
		require_logger.logger_default.error(`Failed to load function: ${err.message}`);
		throw err;
	}
}
/**
* Extracts the file path and function name from a file:// URL
* @param fileUrl The file:// URL (e.g., "file://path/to/file.js:functionName")
* @returns The file path and optional function name
*/
function parseFileUrl(fileUrl) {
	if (!fileUrl.startsWith("file://")) throw new Error("URL must start with file://");
	const urlWithoutProtocol = fileUrl.slice(7);
	const lastColonIndex = urlWithoutProtocol.lastIndexOf(":");
	if (lastColonIndex > 1) return {
		filePath: urlWithoutProtocol.slice(0, lastColonIndex),
		functionName: urlWithoutProtocol.slice(lastColonIndex + 1)
	};
	return { filePath: urlWithoutProtocol };
}

//#endregion
//#region src/util/file.ts
/**
* Simple Nunjucks engine specifically for file paths
* This function is separate from the main getNunjucksEngine to avoid circular dependencies
*/
function getNunjucksEngineForFilePath() {
	const env = nunjucks.default.configure({ autoescape: false });
	env.addGlobal("env", {
		...process.env,
		...require_logger.cliState_default.config?.env
	});
	return env;
}
/**
* Loads content from an external file if the input is a file path, otherwise
* returns the input as-is. Supports Nunjucks templating for file paths.
*
* @param filePath - The input to process. Can be a file path string starting with "file://",
* an array of file paths, or any other type of data.
* @param context - Optional context to control file loading behavior. 'assertion' context
* preserves Python/JS file references instead of loading their content.
* @returns The loaded content if the input was a file path, otherwise the original input.
* For JSON and YAML files, the content is parsed into an object.
* For other file types, the raw file content is returned as a string.
*
* @throws {Error} If the specified file does not exist.
*/
function maybeLoadFromExternalFile(filePath, context) {
	if (Array.isArray(filePath)) return filePath.map((path$2) => {
		return maybeLoadFromExternalFile(path$2, context);
	});
	if (typeof filePath !== "string") return filePath;
	if (!filePath.startsWith("file://")) return filePath;
	const renderedFilePath = getNunjucksEngineForFilePath().renderString(filePath, {});
	const { filePath: cleanPath, functionName } = parseFileUrl(renderedFilePath);
	if (context === "assertion" && (cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath))) {
		require_logger.logger_default.debug(`Preserving Python/JS file reference in assertion context: ${renderedFilePath}`);
		return renderedFilePath;
	}
	if (context === "vars") {
		require_logger.logger_default.debug(`Preserving file reference in vars context: ${renderedFilePath}`);
		return renderedFilePath;
	}
	if (functionName && (cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath))) return renderedFilePath;
	const pathToUse = functionName && !(cleanPath.endsWith(".py") || require_fileExtensions.isJavascriptFile(cleanPath)) ? renderedFilePath.slice(7) : cleanPath;
	const resolvedPath = path.resolve(require_logger.cliState_default.basePath || "", pathToUse);
	if ((0, glob.hasMagic)(pathToUse)) {
		const matchedFiles = (0, glob.globSync)(resolvedPath, { windowsPathsNoEscape: true });
		if (matchedFiles.length === 0) throw new Error(`No files found matching pattern: ${resolvedPath}`);
		const allContents = [];
		for (const matchedFile of matchedFiles) {
			const contents$1 = fs.readFileSync(matchedFile, "utf8");
			if (matchedFile.endsWith(".json")) {
				const parsed = JSON.parse(contents$1);
				if (Array.isArray(parsed)) allContents.push(...parsed);
				else allContents.push(parsed);
			} else if (matchedFile.endsWith(".yaml") || matchedFile.endsWith(".yml")) {
				const parsed = js_yaml.default.load(contents$1);
				if (parsed === null || parsed === void 0) continue;
				if (Array.isArray(parsed)) allContents.push(...parsed);
				else allContents.push(parsed);
			} else if (matchedFile.endsWith(".csv")) {
				const records = (0, csv_parse_sync.parse)(contents$1, { columns: true });
				if (records.length > 0 && Object.keys(records[0]).length === 1) allContents.push(...records.map((record) => Object.values(record)[0]));
				else allContents.push(...records);
			} else allContents.push(contents$1);
		}
		return allContents;
	}
	const finalPath = resolvedPath;
	if (!fs.existsSync(finalPath)) throw new Error(`File does not exist: ${finalPath}`);
	let contents;
	try {
		contents = fs.readFileSync(finalPath, "utf8");
	} catch (error) {
		throw new Error(`Failed to read file ${finalPath}: ${error}`);
	}
	if (finalPath.endsWith(".json")) try {
		return JSON.parse(contents);
	} catch (error) {
		throw new Error(`Failed to parse JSON file ${finalPath}: ${error}`);
	}
	if (finalPath.endsWith(".yaml") || finalPath.endsWith(".yml")) try {
		return js_yaml.default.load(contents);
	} catch (error) {
		throw new Error(`Failed to parse YAML file ${finalPath}: ${error}`);
	}
	if (finalPath.endsWith(".csv")) {
		const records = (0, csv_parse_sync.parse)(contents, { columns: true });
		if (records.length > 0 && Object.keys(records[0]).length === 1) return records.map((record) => Object.values(record)[0]);
		return records;
	}
	return contents;
}
/**
* Resolves a relative file path with respect to a base path, handling cloud configuration appropriately.
* When using a cloud configuration, the current working directory is always used instead of the context's base path.
*
* @param filePath - The relative or absolute file path to resolve.
* @param isCloudConfig - Whether this is a cloud configuration.
* @returns The resolved absolute file path.
*/
function getResolvedRelativePath(filePath, isCloudConfig) {
	if (path.isAbsolute(filePath) || !isCloudConfig) return filePath;
	return path.join(process.cwd(), filePath);
}
/**
* Recursively loads external file references from a configuration object.
*
* @param config - The configuration object to process
* @param context - Optional context to control file loading behavior
* @returns The configuration with external file references resolved
*/
function maybeLoadConfigFromExternalFile(config, context) {
	if (Array.isArray(config)) return config.map((item) => maybeLoadConfigFromExternalFile(item, context));
	if (config && typeof config === "object" && config !== null) {
		const result = {};
		for (const key of Object.keys(config)) {
			const childContext = key === "value" && typeof config === "object" && config && "type" in config && typeof config.type === "string" && (config.type === "python" || config.type === "javascript") ? "assertion" : key === "vars" ? "vars" : context;
			result[key] = maybeLoadConfigFromExternalFile(config[key], childContext);
		}
		return result;
	}
	return maybeLoadFromExternalFile(config, context);
}

//#endregion
//#region src/util/templates.ts
/**
* Get a Nunjucks engine instance with optional filters and configuration.
* @param filters - Optional map of custom Nunjucks filters.
* @param throwOnUndefined - Whether to throw an error on undefined variables.
* @param isGrader - Whether this engine is being used in a grader context.
* Nunjucks is always enabled in grader mode.
* @returns A configured Nunjucks environment.
*/
function getNunjucksEngine(filters, throwOnUndefined = false, isGrader = false) {
	if (!isGrader && require_logger.getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return { renderString: (template) => template };
	const env = nunjucks.default.configure({
		autoescape: false,
		throwOnUndefined
	});
	const envGlobals = {
		...require_logger.getEnvBool("PROMPTFOO_DISABLE_TEMPLATE_ENV_VARS", require_logger.getEnvBool("PROMPTFOO_SELF_HOSTED", false)) ? {} : process.env,
		...require_logger.cliState_default.config?.env
	};
	env.addGlobal("env", envGlobals);
	env.addFilter("load", function(str) {
		return JSON.parse(str);
	});
	if (filters) for (const [name, filter] of Object.entries(filters)) env.addFilter(name, filter);
	return env;
}
/**
* Parse Nunjucks template to extract variables.
* @param template - The Nunjucks template string.
* @returns An array of variables used in the template.
*/
function extractVariablesFromTemplate(template) {
	const variableSet = /* @__PURE__ */ new Set();
	const regex = /\{\{[\s]*([^{}\s|]+)[\s]*(?:\|[^}]+)?\}\}|\{%[\s]*(?:if|for)[\s]+([^{}\s]+)[\s]*.*?%\}/g;
	template = template.replace(/\{#[\s\S]*?#\}/g, "");
	let match;
	while ((match = regex.exec(template)) !== null) {
		const variable = match[1] || match[2];
		if (variable) variableSet.add(variable);
	}
	const forLoopRegex = /\{%[\s]*for[\s]+(\w+)[\s]+in[\s]+(\w+)[\s]*%\}/g;
	while ((match = forLoopRegex.exec(template)) !== null) {
		variableSet.delete(match[1]);
		variableSet.add(match[2]);
	}
	return Array.from(variableSet);
}
/**
* Extract variables from multiple Nunjucks templates.
* @param templates - An array of Nunjucks template strings.
* @returns An array of variables used in the templates.
*/
function extractVariablesFromTemplates(templates) {
	const variableSet = /* @__PURE__ */ new Set();
	for (const template of templates) extractVariablesFromTemplate(template).forEach((variable) => variableSet.add(variable));
	return Array.from(variableSet);
}

//#endregion
//#region src/util/index.ts
const outputToSimpleString = (output) => {
	const passFailText = output.pass ? "[PASS]" : output.failureReason === require_types.ResultFailureReason.ASSERT ? "[FAIL]" : "[ERROR]";
	const namedScoresText = Object.entries(output.namedScores).map(([name, value]) => `${name}: ${value?.toFixed(2)}`).join(", ");
	const scoreText = namedScoresText.length > 0 ? `(${output.score?.toFixed(2)}, ${namedScoresText})` : `(${output.score?.toFixed(2)})`;
	const gradingResultText = output.gradingResult ? `${output.pass ? "Pass" : "Fail"} Reason: ${output.gradingResult.reason}` : "";
	return dedent.default`
      ${passFailText} ${scoreText}

      ${output.text}

      ${gradingResultText}
    `.trim();
};
function createOutputMetadata(evalRecord) {
	let evaluationCreatedAt;
	if (evalRecord.createdAt) try {
		const date = new Date(evalRecord.createdAt);
		evaluationCreatedAt = Number.isNaN(date.getTime()) ? void 0 : date.toISOString();
	} catch {
		evaluationCreatedAt = void 0;
	}
	return {
		promptfooVersion: require_fetch.VERSION,
		nodeVersion: process.version,
		platform: os.platform(),
		arch: os.arch(),
		exportedAt: (/* @__PURE__ */ new Date()).toISOString(),
		evaluationCreatedAt,
		author: evalRecord.author
	};
}
/**
* JSON writer with improved error handling for large datasets.
* Provides helpful error messages when memory limits are exceeded.
*/
async function writeJsonOutputSafely(outputPath, evalRecord, shareableUrl) {
	const metadata = createOutputMetadata(evalRecord);
	try {
		const summary = await evalRecord.toEvaluateSummary();
		const outputData = {
			evalId: evalRecord.id,
			results: summary,
			config: evalRecord.config,
			shareableUrl,
			metadata
		};
		const jsonString = JSON.stringify(outputData, null, 2);
		fs.writeFileSync(outputPath, jsonString);
	} catch (error) {
		const msg = error?.message ?? "";
		const isStringLen = error instanceof RangeError && msg.includes("Invalid string length");
		const isHeapOOM = /heap out of memory|Array buffer allocation failed|ERR_STRING_TOO_LONG/i.test(msg);
		if (isStringLen || isHeapOOM) {
			const resultCount = await evalRecord.getResultsCount();
			require_logger.logger_default.error(`Dataset too large for JSON export (${resultCount} results).`);
			throw new Error(`Dataset too large for JSON export. The evaluation has ${resultCount} results which exceeds memory limits. Consider using JSONL format instead: --output output.jsonl`);
		} else throw error;
	}
}
async function writeOutput(outputPath, evalRecord, shareableUrl) {
	if (outputPath.match(/^https:\/\/docs\.google\.com\/spreadsheets\//)) {
		const table = await evalRecord.getTable();
		require_invariant.invariant(table, "Table is required");
		const rows = table.body.map((row) => {
			const csvRow = {};
			table.head.vars.forEach((varName, index) => {
				csvRow[varName] = row.vars[index];
			});
			table.head.prompts.forEach((prompt, index) => {
				csvRow[`[${prompt.provider}] ${prompt.label}`] = outputToSimpleString(row.outputs[index]);
			});
			return csvRow;
		});
		require_logger.logger_default.info(`Writing ${rows.length} rows to Google Sheets...`);
		await writeCsvToGoogleSheet(rows, outputPath);
		return;
	}
	const { data: outputExtension } = require_types.OutputFileExtension.safeParse(path.extname(outputPath).slice(1).toLowerCase());
	require_invariant.invariant(outputExtension, `Unsupported output file format ${outputExtension}. Please use one of: ${require_types.OutputFileExtension.options.join(", ")}.`);
	const outputDir = path.dirname(outputPath);
	if (!fs.existsSync(outputDir)) fs.mkdirSync(outputDir, { recursive: true });
	const metadata = createOutputMetadata(evalRecord);
	if (outputExtension === "csv") {
		const headers = getHeaderForTable(evalRecord);
		const headerCsv = (0, csv_stringify_sync.stringify)([[...headers.vars, ...headers.prompts.map((prompt) => `[${prompt.provider}] ${prompt.label}`)]]);
		fs.writeFileSync(outputPath, headerCsv);
		for await (const batchResults of evalRecord.fetchResultsBatched()) {
			const tableRows = {};
			for (const result of batchResults) {
				if (!(result.testIdx in tableRows)) tableRows[result.testIdx] = [];
				tableRows[result.testIdx].push(result);
			}
			const batchCsv = (0, csv_stringify_sync.stringify)(Object.values(tableRows).map((results) => {
				const row = convertTestResultsToTableRow(results, headers.vars);
				return [...row.vars, ...row.outputs.map(outputToSimpleString)];
			}));
			fs.appendFileSync(outputPath, batchCsv);
		}
	} else if (outputExtension === "json") await writeJsonOutputSafely(outputPath, evalRecord, shareableUrl);
	else if (outputExtension === "yaml" || outputExtension === "yml" || outputExtension === "txt") {
		const summary = await evalRecord.toEvaluateSummary();
		fs.writeFileSync(outputPath, js_yaml.default.dump({
			evalId: evalRecord.id,
			results: summary,
			config: evalRecord.config,
			shareableUrl,
			metadata
		}));
	} else if (outputExtension === "html") {
		const table = await evalRecord.getTable();
		require_invariant.invariant(table, "Table is required");
		const summary = await evalRecord.toEvaluateSummary();
		const template = fs.readFileSync(path.join(require_esm.getDirectory(), "tableOutput.html"), "utf-8");
		const htmlTable = [[...table.head.vars, ...table.head.prompts.map((prompt) => `[${prompt.provider}] ${prompt.label}`)], ...table.body.map((row) => [...row.vars, ...row.outputs.map(outputToSimpleString)])];
		const htmlOutput = getNunjucksEngine().renderString(template, {
			config: evalRecord.config,
			table: htmlTable,
			results: summary
		});
		fs.writeFileSync(outputPath, htmlOutput);
	} else if (outputExtension === "jsonl") for await (const batchResults of evalRecord.fetchResultsBatched()) {
		const text = batchResults.map((result) => JSON.stringify(result)).join(os.EOL) + os.EOL;
		fs.appendFileSync(outputPath, text);
	}
	else if (outputExtension === "xml") {
		const summary = await evalRecord.toEvaluateSummary();
		const sanitizeForXml = (obj) => {
			if (obj === null || obj === void 0) return "";
			if (typeof obj === "boolean" || typeof obj === "number") return String(obj);
			if (typeof obj === "string") return obj;
			if (Array.isArray(obj)) return obj.map(sanitizeForXml);
			if (typeof obj === "object") {
				const sanitized = {};
				for (const [key, value] of Object.entries(obj)) sanitized[key] = sanitizeForXml(value);
				return sanitized;
			}
			return String(obj);
		};
		const xmlData = new fast_xml_parser.XMLBuilder({
			ignoreAttributes: false,
			format: true,
			indentBy: "  "
		}).build({ promptfoo: {
			evalId: evalRecord.id,
			results: sanitizeForXml(summary),
			config: sanitizeForXml(evalRecord.config),
			shareableUrl: shareableUrl || ""
		} });
		fs.writeFileSync(outputPath, xmlData);
	}
}
async function writeMultipleOutputs(outputPaths, evalRecord, shareableUrl) {
	await Promise.all(outputPaths.map((outputPath) => writeOutput(outputPath, evalRecord, shareableUrl)));
}
async function readOutput(outputPath) {
	const ext = path.parse(outputPath).ext.slice(1);
	switch (ext) {
		case "json": return JSON.parse(fs.readFileSync(outputPath, "utf-8"));
		default: throw new Error(`Unsupported output file format: ${ext} currently only supports json`);
	}
}
async function readFilters(filters, basePath = "") {
	const ret = {};
	for (const [name, filterPath] of Object.entries(filters)) {
		const filePaths = (0, glob.globSync)(path.join(basePath, filterPath), { windowsPathsNoEscape: true });
		for (const filePath of filePaths) ret[name] = await require_esm.importModule(path.resolve(filePath));
	}
	return ret;
}
function printBorder() {
	const border = "=".repeat(require_fetch.TERMINAL_MAX_WIDTH);
	require_logger.logger_default.info(border);
}
/**
* Load environment variables from .env file(s).
* @param envPath - Single path, array of paths, or undefined for default .env loading.
*                  When paths are explicitly specified, all files must exist or an error is thrown.
*                  When multiple files are provided, later files override values from earlier files.
*/
function setupEnv(envPath) {
	if (envPath) {
		const paths = (Array.isArray(envPath) ? envPath : [envPath]).flatMap((p) => p.includes(",") ? p.split(",").map((s) => s.trim()) : p.trim()).filter((p) => p.length > 0);
		if (paths.length === 0) {
			dotenv.default.config({ quiet: true });
			return;
		}
		for (const p of paths) if (!fs.existsSync(p)) throw new Error(`Environment file not found: ${p}`);
		if (paths.length === 1) require_logger.logger_default.info(`Loading environment variables from ${paths[0]}`);
		else require_logger.logger_default.info(`Loading environment variables from: ${paths.join(", ")}`);
		const pathArg = paths.length === 1 ? paths[0] : paths;
		dotenv.default.config({
			path: pathArg,
			override: true,
			quiet: true
		});
	} else dotenv.default.config({ quiet: true });
}
function canonicalizeProviderId(id) {
	if (id.startsWith("file://")) {
		const filePath = id.slice(7);
		return path.isAbsolute(filePath) ? id : `file://${path.resolve(filePath)}`;
	}
	for (const prefix of [
		"exec:",
		"python:",
		"golang:"
	]) if (id.startsWith(prefix)) {
		const filePath = id.slice(prefix.length);
		if (filePath.includes("/") || filePath.includes("\\")) return `${prefix}${path.resolve(filePath)}`;
		return id;
	}
	if ((id.endsWith(".js") || id.endsWith(".ts") || id.endsWith(".mjs")) && (id.includes("/") || id.includes("\\"))) return `file://${path.resolve(id)}`;
	return id;
}
function getProviderLabel(provider) {
	return provider?.label && typeof provider.label === "string" ? provider.label : void 0;
}
function providerToIdentifier(provider) {
	if (!provider) return;
	if (typeof provider === "string") return canonicalizeProviderId(provider);
	const label = getProviderLabel(provider);
	if (label) return label;
	if (require_types.isApiProvider(provider)) return canonicalizeProviderId(provider.id());
	if (require_types.isProviderOptions(provider)) {
		if (provider.id) return canonicalizeProviderId(provider.id);
		return;
	}
	if (typeof provider === "object" && "id" in provider && typeof provider.id === "string") return canonicalizeProviderId(provider.id);
}
/**
* Runtime variables that are added during evaluation but aren't part
* of the original test definition. These should be filtered when
* comparing test cases for matching purposes.
*
* - _conversation: Added by the evaluator for multi-turn conversations
* - sessionId: Added by multi-turn strategy providers (GOAT, Crescendo, SIMBA)
*/
const RUNTIME_VAR_KEYS = ["_conversation", "sessionId"];
/**
* Filters out runtime-only variables that are added during evaluation
* but aren't part of the original test definition.
*
* This is used when comparing test cases to determine if a result
* corresponds to a particular test, regardless of runtime state.
*/
function filterRuntimeVars(vars) {
	if (!vars) return vars;
	const filtered = { ...vars };
	for (const key of RUNTIME_VAR_KEYS) delete filtered[key];
	return filtered;
}
function varsMatch(vars1, vars2) {
	return (0, fast_deep_equal.default)(vars1, vars2);
}
function resultIsForTestCase(result, testCase) {
	const providersMatch = testCase.provider ? providerToIdentifier(testCase.provider) === providerToIdentifier(result.provider) : true;
	const resultVars = filterRuntimeVars(result.vars);
	return varsMatch(filterRuntimeVars(testCase.vars), resultVars) && providersMatch;
}
/**
* Renders ONLY environment variable templates in an object, leaving all other templates untouched.
* This allows env vars to be resolved at provider load time while preserving runtime var templates.
*
* Supports full Nunjucks syntax for env vars including filters and expressions:
* - {{ env.VAR_NAME }}
* - {{ env['VAR-NAME'] }}
* - {{ env["VAR-NAME"] }}
* - {{ env.VAR | default('fallback') }}
* - {{ env.VAR | upper }}
*
* Preserves non-env templates for runtime rendering:
* - {{ vars.x }} - preserved as literal
* - {{ prompt }} - preserved as literal
*
* Implementation: Uses regex to find env templates, delegates to Nunjucks for rendering.
* This ensures full Nunjucks feature support while preserving non-env templates.
*
* @param obj - The object to process
* @returns The object with only env templates rendered
*/
function renderEnvOnlyInObject(obj) {
	if (require_logger.getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return obj;
	if (typeof obj === "string") {
		if (obj.length > 5e4) {
			require_logger.logger_default.warn(`String too long (${obj.length} chars) for template matching. Skipping env var rendering.`);
			return obj;
		}
		const nunjucks$3 = getNunjucksEngine();
		const envGlobals = nunjucks$3.getGlobal("env");
		return obj.replace(/\{\{(?:[^}]|\}(?!\}))*\}\}/g, (match) => {
			if (!match.match(/\benv\.|env\[/)) return match;
			const varMatch = match.match(/env\.(\w+)|env\[['"]([^'"]+)['"]\]/);
			const varName = varMatch?.[1] || varMatch?.[2];
			if (match.includes("|") || varName && varName in envGlobals) try {
				return nunjucks$3.renderString(match, { env: envGlobals });
			} catch (_error) {
				return match;
			}
			return match;
		});
	}
	if (Array.isArray(obj)) return obj.map((item) => renderEnvOnlyInObject(item));
	if (typeof obj === "object" && obj !== null) {
		const result = {};
		for (const key in obj) result[key] = renderEnvOnlyInObject(obj[key]);
		return result;
	}
	return obj;
}
function renderVarsInObject(obj, vars) {
	if (!vars || require_logger.getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return obj;
	if (typeof obj === "string") return getNunjucksEngine().renderString(obj, vars);
	if (Array.isArray(obj)) return obj.map((item) => renderVarsInObject(item, vars));
	if (typeof obj === "object" && obj !== null) {
		const result = {};
		for (const key in obj) result[key] = renderVarsInObject(obj[key], vars);
		return result;
	} else if (typeof obj === "function") return renderVarsInObject(obj({ vars }));
	return obj;
}
/**
* Parses a file path or glob pattern to extract function names and file extensions.
* Function names can be specified in the filename like this:
* prompt.py:myFunction or prompts.js:myFunction.
* @param basePath - The base path for file resolution.
* @param promptPath - The path or glob pattern.
* @returns Parsed details including function name, file extension, and directory status.
*/
function parsePathOrGlob(basePath, promptPath) {
	if (promptPath.startsWith("file://")) promptPath = promptPath.slice(7);
	const filePath = path.resolve(basePath, promptPath);
	let filename = path.relative(basePath, filePath);
	let functionName;
	if (filename.includes(":")) {
		const lastColonIndex = filename.lastIndexOf(":");
		if (lastColonIndex > 1) {
			const pathWithoutFunction = filename.slice(0, lastColonIndex);
			if (require_fileExtensions.isJavascriptFile(pathWithoutFunction) || pathWithoutFunction.endsWith(".py") || pathWithoutFunction.endsWith(".go") || pathWithoutFunction.endsWith(".rb")) {
				functionName = filename.slice(lastColonIndex + 1);
				filename = pathWithoutFunction;
			}
		}
	}
	let stats;
	try {
		stats = fs.statSync(path.join(basePath, filename));
	} catch (err) {
		if (require_logger.getEnvBool("PROMPTFOO_STRICT_FILES")) throw err;
	}
	const normalizedFilePath = filePath.replace(/\\/g, "/");
	const isPathPattern = stats?.isDirectory() || (0, glob.hasMagic)(promptPath) || (0, glob.hasMagic)(normalizedFilePath);
	const safeFilename = path.relative(basePath, require_esm.safeResolve(basePath, filename));
	return {
		extension: isPathPattern ? void 0 : path.parse(safeFilename).ext,
		filePath: safeFilename.startsWith(basePath) ? safeFilename : path.join(basePath, safeFilename),
		functionName,
		isPathPattern
	};
}
/**
* Renders variables in a tools object and loads from external file if applicable.
* This function combines renderVarsInObject and maybeLoadFromExternalFile into a single step
* specifically for handling tools configurations.
*
* Supports loading from JSON, YAML, Python, and JavaScript files.
*
* @param tools - The tools configuration object or array to process.
* @param vars - Variables to use for rendering.
* @returns The processed tools configuration with variables rendered and content loaded from files if needed.
* @throws {Error} If the loaded tools are in an invalid format
*/
async function maybeLoadToolsFromExternalFile(tools, vars) {
	const rendered = renderVarsInObject(tools, vars);
	if (typeof rendered === "string" && rendered.startsWith("file://")) {
		const { filePath, functionName } = parseFileUrl(rendered);
		if (functionName && (filePath.endsWith(".py") || require_fileExtensions.isJavascriptFile(filePath))) {
			const fileType = filePath.endsWith(".py") ? "Python" : "JavaScript";
			require_logger.logger_default.debug(`[maybeLoadToolsFromExternalFile] Loading tools from ${fileType} file: ${filePath}:${functionName}`);
			try {
				let toolDefinitions;
				if (filePath.endsWith(".py")) {
					const absPath = require_esm.safeResolve(require_logger.cliState_default.basePath || process.cwd(), filePath);
					require_logger.logger_default.debug(`[maybeLoadToolsFromExternalFile] Resolved Python path: ${absPath}`);
					toolDefinitions = await require_pythonUtils.runPython(absPath, functionName, []);
				} else {
					const absPath = require_esm.safeResolve(require_logger.cliState_default.basePath || process.cwd(), filePath);
					require_logger.logger_default.debug(`[maybeLoadToolsFromExternalFile] Resolved JavaScript path: ${absPath}`);
					const module$1 = await require_esm.importModule(absPath);
					const fn = module$1[functionName] || module$1.default?.[functionName];
					if (typeof fn !== "function") {
						const availableExports = Object.keys(module$1).filter((k) => k !== "default");
						const basePath = require_logger.cliState_default.basePath || process.cwd();
						throw new Error(`Function "${functionName}" not found in ${filePath}. Available exports: ${availableExports.length > 0 ? availableExports.join(", ") : "(none)"}\nResolved from: ${basePath}`);
					}
					toolDefinitions = await Promise.resolve(fn());
				}
				if (!toolDefinitions || typeof toolDefinitions === "string" || typeof toolDefinitions === "number" || typeof toolDefinitions === "boolean") throw new Error(`Function "${functionName}" must return an array or object of tool definitions, but returned: ${toolDefinitions === null ? "null" : typeof toolDefinitions}`);
				require_logger.logger_default.debug(`[maybeLoadToolsFromExternalFile] Successfully loaded ${Array.isArray(toolDefinitions) ? toolDefinitions.length : "object"} tools`);
				return toolDefinitions;
			} catch (err) {
				const errorMessage = err instanceof Error ? err.message : String(err);
				const basePath = require_logger.cliState_default.basePath || process.cwd();
				throw new Error(`Failed to load tools from ${rendered}:\n${errorMessage}\n\nMake sure the function "${functionName}" exists and returns a valid tool definition array.\nResolved from: ${basePath}`);
			}
		}
		if (filePath.endsWith(".py") || require_fileExtensions.isJavascriptFile(filePath)) {
			const ext = filePath.endsWith(".py") ? "Python" : "JavaScript";
			const basePath = require_logger.cliState_default.basePath || process.cwd();
			throw new Error(`Cannot load tools from ${rendered}\n${ext} files require a function name. Use this format:\n  tools: file://${filePath}:get_tools\n\nYour ${ext} file should export a function that returns tool definitions:\n` + (filePath.endsWith(".py") ? `  def get_tools():\n      return [{"type": "function", "function": {...}}]` : `  module.exports.get_tools = () => [{ type: "function", function: {...} }];`) + `\n\nResolved from: ${basePath}`);
		}
	}
	if (Array.isArray(rendered)) {
		const results = await Promise.all(rendered.map((item) => maybeLoadToolsFromExternalFile(item, vars)));
		if (results.every((r) => Array.isArray(r))) return results.flat();
		return results;
	}
	if (typeof rendered !== "string") return rendered;
	const loaded = maybeLoadFromExternalFile(rendered);
	if (loaded !== void 0 && loaded !== null && typeof loaded === "string") {
		if (loaded.startsWith("file://")) throw new Error(`Failed to load tools from ${loaded}\nEnsure the file exists and contains valid JSON or YAML tool definitions.`);
		if (loaded.includes("def ") || loaded.includes("import ")) throw new Error("Invalid tools configuration: file appears to contain Python code.\nPython files require a function name. Use this format:\n  tools: file://tools.py:get_tools");
		throw new Error("Invalid tools configuration: expected an array or object, but got a string.\nIf using file://, ensure the file contains valid JSON or YAML tool definitions.");
	}
	return loaded;
}

//#endregion
//#region src/providers/openai/util.ts
const ajv = require_logger.getAjv();
const OPENAI_CHAT_MODELS = [
	{
		id: "gpt-4o-mini-tts",
		cost: {
			input: .6 / 1e6,
			output: 0 / 1e6,
			audioOutput: 12 / 1e6
		}
	},
	...["gpt-4o-search-preview", "gpt-4o-search-preview-2025-03-11"].map((model) => ({
		id: model,
		cost: {
			input: 2.5 / 1e6,
			output: 10 / 1e6
		}
	})),
	...["gpt-4o-mini-search-preview", "gpt-4o-mini-search-preview-2025-03-11"].map((model) => ({
		id: model,
		cost: {
			input: .15 / 1e6,
			output: .6 / 1e6
		}
	})),
	...["computer-use-preview", "computer-use-preview-2025-03-11"].map((model) => ({
		id: model,
		cost: {
			input: 3 / 1e6,
			output: 12 / 1e6
		}
	})),
	...["chatgpt-4o-latest"].map((model) => ({
		id: model,
		cost: {
			input: 5 / 1e6,
			output: 15 / 1e6
		}
	})),
	...["gpt-4.1", "gpt-4.1-2025-04-14"].map((model) => ({
		id: model,
		cost: {
			input: 2 / 1e6,
			output: 8 / 1e6
		}
	})),
	...["gpt-4.1-mini", "gpt-4.1-mini-2025-04-14"].map((model) => ({
		id: model,
		cost: {
			input: .4 / 1e6,
			output: 1.6 / 1e6
		}
	})),
	...["gpt-4.1-nano", "gpt-4.1-nano-2025-04-14"].map((model) => ({
		id: model,
		cost: {
			input: .1 / 1e6,
			output: .4 / 1e6
		}
	})),
	...["o1-pro", "o1-pro-2025-03-19"].map((model) => ({
		id: model,
		cost: {
			input: 150 / 1e6,
			output: 600 / 1e6
		}
	})),
	...[
		"o1",
		"o1-2024-12-17",
		"o1-preview",
		"o1-preview-2024-09-12"
	].map((model) => ({
		id: model,
		cost: {
			input: 15 / 1e6,
			output: 60 / 1e6
		}
	})),
	...["o1-mini", "o1-mini-2024-09-12"].map((model) => ({
		id: model,
		cost: {
			input: 1.1 / 1e6,
			output: 4.4 / 1e6
		}
	})),
	...["o3", "o3-2025-04-16"].map((model) => ({
		id: model,
		cost: {
			input: 2 / 1e6,
			output: 8 / 1e6
		}
	})),
	...["o3-pro", "o3-pro-2025-06-10"].map((model) => ({
		id: model,
		cost: {
			input: 20 / 1e6,
			output: 80 / 1e6
		}
	})),
	...["o3-mini", "o3-mini-2025-01-31"].map((model) => ({
		id: model,
		cost: {
			input: 1.1 / 1e6,
			output: 4.4 / 1e6
		}
	})),
	...[
		"gpt-4o-audio-preview",
		"gpt-4o-audio-preview-2024-12-17",
		"gpt-4o-audio-preview-2024-10-01",
		"gpt-4o-audio-preview-2025-06-03"
	].map((model) => ({
		id: model,
		cost: {
			input: 2.5 / 1e6,
			output: 10 / 1e6,
			audioInput: 40 / 1e6,
			audioOutput: 80 / 1e6
		}
	})),
	...["gpt-4o-mini-audio-preview", "gpt-4o-mini-audio-preview-2024-12-17"].map((model) => ({
		id: model,
		cost: {
			input: .15 / 1e6,
			output: .6 / 1e6,
			audioInput: 10 / 1e6,
			audioOutput: 20 / 1e6
		}
	})),
	...[
		"gpt-4o",
		"gpt-4o-2024-11-20",
		"gpt-4o-2024-08-06"
	].map((model) => ({
		id: model,
		cost: {
			input: 2.5 / 1e6,
			output: 10 / 1e6
		}
	})),
	...["gpt-4o-2024-05-13"].map((model) => ({
		id: model,
		cost: {
			input: 5 / 1e6,
			output: 15 / 1e6
		}
	})),
	...["gpt-4o-mini", "gpt-4o-mini-2024-07-18"].map((model) => ({
		id: model,
		cost: {
			input: .15 / 1e6,
			output: .6 / 1e6
		}
	})),
	...[
		"gpt-4",
		"gpt-4-0613",
		"gpt-4-0314"
	].map((model) => ({
		id: model,
		cost: {
			input: 30 / 1e6,
			output: 60 / 1e6
		}
	})),
	...[
		"gpt-4-32k",
		"gpt-4-32k-0314",
		"gpt-4-32k-0613"
	].map((model) => ({
		id: model,
		cost: {
			input: 60 / 1e6,
			output: 120 / 1e6
		}
	})),
	...[
		"gpt-4-turbo",
		"gpt-4-turbo-2024-04-09",
		"gpt-4-turbo-preview",
		"gpt-4-0125-preview",
		"gpt-4-1106-preview",
		"gpt-4-1106-vision-preview",
		"gpt-4-vision-preview"
	].map((model) => ({
		id: model,
		cost: {
			input: 10 / 1e6,
			output: 30 / 1e6
		}
	})),
	{
		id: "gpt-3.5-turbo",
		cost: {
			input: .5 / 1e6,
			output: 1.5 / 1e6
		}
	},
	{
		id: "gpt-3.5-turbo-0125",
		cost: {
			input: .5 / 1e6,
			output: 1.5 / 1e6
		}
	},
	{
		id: "gpt-3.5-turbo-1106",
		cost: {
			input: 1 / 1e6,
			output: 2 / 1e6
		}
	},
	...["gpt-3.5-turbo-0301", "gpt-3.5-turbo-0613"].map((model) => ({
		id: model,
		cost: {
			input: 1.5 / 1e6,
			output: 2 / 1e6
		}
	})),
	...["gpt-3.5-turbo-16k", "gpt-3.5-turbo-16k-0613"].map((model) => ({
		id: model,
		cost: {
			input: 3 / 1e6,
			output: 4 / 1e6
		}
	})),
	...["gpt-3.5-turbo-instruct"].map((model) => ({
		id: model,
		cost: {
			input: 1.5 / 1e6,
			output: 2 / 1e6
		}
	})),
	...["o4-mini", "o4-mini-2025-04-16"].map((model) => ({
		id: model,
		cost: {
			input: 1.1 / 1e6,
			output: 4.4 / 1e6
		}
	})),
	...["gpt-5", "gpt-5-2025-08-07"].map((model) => ({
		id: model,
		cost: {
			input: 1.25 / 1e6,
			output: 10 / 1e6
		}
	})),
	...["gpt-5-chat-latest"].map((model) => ({
		id: model,
		cost: {
			input: 1.25 / 1e6,
			output: 10 / 1e6
		}
	})),
	...["gpt-5-nano", "gpt-5-nano-2025-08-07"].map((model) => ({
		id: model,
		cost: {
			input: .05 / 1e6,
			output: .4 / 1e6
		}
	})),
	...["gpt-5-mini", "gpt-5-mini-2025-08-07"].map((model) => ({
		id: model,
		cost: {
			input: .25 / 1e6,
			output: 2 / 1e6
		}
	})),
	...["codex-mini-latest"].map((model) => ({
		id: model,
		cost: {
			input: 1.5 / 1e6,
			output: 6 / 1e6
		}
	})),
	...["gpt-5-codex"].map((model) => ({
		id: model,
		cost: {
			input: 1.25 / 1e6,
			output: 10 / 1e6
		}
	})),
	...["gpt-5-pro", "gpt-5-pro-2025-10-06"].map((model) => ({
		id: model,
		cost: {
			input: 15 / 1e6,
			output: 120 / 1e6
		}
	})),
	...["gpt-5.1"].map((model) => ({
		id: model,
		cost: {
			input: 1.25 / 1e6,
			output: 10 / 1e6
		}
	})),
	...["gpt-5.1-nano"].map((model) => ({
		id: model,
		cost: {
			input: .05 / 1e6,
			output: .4 / 1e6
		}
	})),
	...["gpt-5.1-mini"].map((model) => ({
		id: model,
		cost: {
			input: .25 / 1e6,
			output: 2 / 1e6
		}
	})),
	...["gpt-5.1-codex", "gpt-5.1-codex-max"].map((model) => ({
		id: model,
		cost: {
			input: 1.25 / 1e6,
			output: 10 / 1e6
		}
	})),
	...["gpt-5.2", "gpt-5.2-2025-12-11"].map((model) => ({
		id: model,
		cost: {
			input: 1.75 / 1e6,
			output: 14 / 1e6
		}
	})),
	...["gpt-audio", "gpt-audio-2025-08-28"].map((model) => ({
		id: model,
		cost: {
			input: 2.5 / 1e6,
			output: 10 / 1e6,
			audioInput: 40 / 1e6,
			audioOutput: 80 / 1e6
		}
	})),
	...["gpt-audio-mini", "gpt-audio-mini-2025-10-06"].map((model) => ({
		id: model,
		cost: {
			input: .6 / 1e6,
			output: 2.4 / 1e6,
			audioInput: 10 / 1e6,
			audioOutput: 20 / 1e6
		}
	}))
];
const OPENAI_DEEP_RESEARCH_MODELS = [...["o3-deep-research", "o3-deep-research-2025-06-26"].map((model) => ({
	id: model,
	cost: {
		input: 10 / 1e6,
		output: 40 / 1e6
	}
})), ...["o4-mini-deep-research", "o4-mini-deep-research-2025-06-26"].map((model) => ({
	id: model,
	cost: {
		input: 2 / 1e6,
		output: 8 / 1e6
	}
}))];
const OPENAI_COMPLETION_MODELS = [
	{
		id: "gpt-3.5-turbo-instruct",
		cost: {
			input: 1.5 / 1e6,
			output: 2 / 1e6
		}
	},
	{ id: "text-davinci-002" },
	{ id: "text-babbage-002" }
];
const OPENAI_REALTIME_MODELS = [
	{
		id: "gpt-realtime",
		type: "chat",
		cost: {
			input: 32 / 1e6,
			output: 64 / 1e6,
			audioInput: 32 / 1e6,
			audioOutput: 64 / 1e6
		}
	},
	{
		id: "gpt-realtime",
		type: "chat",
		cost: {
			input: 4 / 1e6,
			output: 16 / 1e6,
			audioInput: 40 / 1e6,
			audioOutput: 80 / 1e6
		}
	},
	{
		id: "gpt-4o-realtime-preview",
		type: "chat",
		cost: {
			input: 5 / 1e6,
			output: 20 / 1e6,
			audioInput: 40 / 1e6,
			audioOutput: 80 / 1e6
		}
	},
	{
		id: "gpt-4o-realtime-preview-2024-12-17",
		type: "chat",
		cost: {
			input: 5 / 1e6,
			output: 20 / 1e6,
			audioInput: 40 / 1e6,
			audioOutput: 80 / 1e6
		}
	},
	{
		id: "gpt-4o-realtime-preview-2024-10-01",
		type: "chat",
		cost: {
			input: 5 / 1e6,
			output: 20 / 1e6,
			audioInput: 100 / 1e6,
			audioOutput: 200 / 1e6
		}
	},
	{
		id: "gpt-4o-mini-realtime-preview",
		type: "chat",
		cost: {
			input: .6 / 1e6,
			output: 2.4 / 1e6,
			audioInput: 10 / 1e6,
			audioOutput: 20 / 1e6
		}
	},
	{
		id: "gpt-4o-mini-realtime-preview-2024-12-17",
		type: "chat",
		cost: {
			input: .6 / 1e6,
			output: 2.4 / 1e6,
			audioInput: 10 / 1e6,
			audioOutput: 20 / 1e6
		}
	},
	{
		id: "gpt-realtime-mini",
		type: "chat",
		cost: {
			input: .6 / 1e6,
			output: 2.4 / 1e6,
			audioInput: 10 / 1e6,
			audioOutput: 20 / 1e6
		}
	},
	{
		id: "gpt-realtime-mini-2025-10-06",
		type: "chat",
		cost: {
			input: .6 / 1e6,
			output: 2.4 / 1e6,
			audioInput: 10 / 1e6,
			audioOutput: 20 / 1e6
		}
	}
];
const OPENAI_TRANSCRIPTION_MODELS = [
	{
		id: "gpt-4o-transcribe",
		cost: { perMinute: .006 }
	},
	{
		id: "gpt-4o-mini-transcribe",
		cost: { perMinute: .003 }
	},
	{
		id: "gpt-4o-transcribe-diarize",
		cost: { perMinute: .006 }
	},
	{
		id: "whisper-1",
		cost: { perMinute: .006 }
	}
];
function calculateOpenAICost(modelName, config, promptTokens, completionTokens, audioPromptTokens, audioCompletionTokens) {
	if (!audioPromptTokens && !audioCompletionTokens) return require_fetch.calculateCost(modelName, config, promptTokens, completionTokens, [
		...OPENAI_CHAT_MODELS,
		...OPENAI_COMPLETION_MODELS,
		...OPENAI_REALTIME_MODELS,
		...OPENAI_DEEP_RESEARCH_MODELS
	]);
	if (!Number.isFinite(promptTokens) || !Number.isFinite(completionTokens) || !Number.isFinite(audioPromptTokens) || !Number.isFinite(audioCompletionTokens) || typeof promptTokens === "undefined" || typeof completionTokens === "undefined" || typeof audioPromptTokens === "undefined" || typeof audioCompletionTokens === "undefined") return;
	const model = [
		...OPENAI_CHAT_MODELS,
		...OPENAI_COMPLETION_MODELS,
		...OPENAI_REALTIME_MODELS,
		...OPENAI_DEEP_RESEARCH_MODELS
	].find((m) => m.id === modelName);
	if (!model || !model.cost) return;
	let totalCost = 0;
	const inputCost = config.cost ?? model.cost.input;
	const outputCost = config.cost ?? model.cost.output;
	totalCost += inputCost * promptTokens + outputCost * completionTokens;
	if ("audioInput" in model.cost || "audioOutput" in model.cost) {
		const audioInputCost = config.audioCost ?? model.cost.audioInput ?? 0;
		const audioOutputCost = config.audioCost ?? model.cost.audioOutput ?? 0;
		totalCost += audioInputCost * audioPromptTokens + audioOutputCost * audioCompletionTokens;
	}
	return totalCost || void 0;
}
function failApiCall(err) {
	if (err instanceof openai.default.APIError) {
		const errorType = err.error?.type || err.type || "unknown";
		const errorMessage = err.error?.message || err.message || "Unknown error";
		return { error: `API error: ${errorType}${err.status ? ` ${err.status}` : ""} ${errorMessage}` };
	}
	return { error: `API error: ${String(err)}` };
}
function getTokenUsage(data, cached) {
	if (data.usage) if (cached) return {
		cached: data.usage.total_tokens,
		total: data.usage.total_tokens,
		numRequests: 1
	};
	else return {
		total: data.usage.total_tokens,
		prompt: data.usage.prompt_tokens || 0,
		completion: data.usage.completion_tokens || 0,
		numRequests: 1,
		...data.usage.completion_tokens_details ? { completionDetails: {
			reasoning: data.usage.completion_tokens_details.reasoning_tokens,
			acceptedPrediction: data.usage.completion_tokens_details.accepted_prediction_tokens,
			rejectedPrediction: data.usage.completion_tokens_details.rejected_prediction_tokens
		} } : {}
	};
	return {};
}
function validateFunctionCall(output, functions, vars) {
	if (typeof output === "object" && "function_call" in output) output = output.function_call;
	const functionCall = output;
	if (typeof functionCall !== "object" || typeof functionCall.name !== "string" || typeof functionCall.arguments !== "string") throw new Error(`OpenAI did not return a valid-looking function call: ${JSON.stringify(functionCall)}`);
	const interpolatedFunctions = maybeLoadFromExternalFile(renderVarsInObject(functions, vars));
	const functionArgs = JSON.parse(functionCall.arguments);
	const functionName = functionCall.name;
	const functionSchema = interpolatedFunctions?.find((f) => f.name === functionName)?.parameters;
	if (!functionSchema) throw new Error(`Called "${functionName}", but there is no function with that name`);
	const validate = ajv.compile(functionSchema);
	if (!validate(functionArgs)) throw new Error(`Call to "${functionName}" does not match schema: ${JSON.stringify(validate.errors)}`);
}
function formatOpenAiError(data) {
	let errorMessage = `API error: ${data.error.message}`;
	if (data.error.type) errorMessage += `, Type: ${data.error.type}`;
	if (data.error.code) errorMessage += `, Code: ${data.error.code}`;
	errorMessage += "\n\n" + require_logger.safeJsonStringify(data, true);
	return errorMessage;
}

//#endregion
Object.defineProperty(exports, 'OPENAI_CHAT_MODELS', {
  enumerable: true,
  get: function () {
    return OPENAI_CHAT_MODELS;
  }
});
Object.defineProperty(exports, 'OPENAI_COMPLETION_MODELS', {
  enumerable: true,
  get: function () {
    return OPENAI_COMPLETION_MODELS;
  }
});
Object.defineProperty(exports, 'OPENAI_REALTIME_MODELS', {
  enumerable: true,
  get: function () {
    return OPENAI_REALTIME_MODELS;
  }
});
Object.defineProperty(exports, 'OPENAI_TRANSCRIPTION_MODELS', {
  enumerable: true,
  get: function () {
    return OPENAI_TRANSCRIPTION_MODELS;
  }
});
Object.defineProperty(exports, 'calculateOpenAICost', {
  enumerable: true,
  get: function () {
    return calculateOpenAICost;
  }
});
Object.defineProperty(exports, 'convertTestResultsToTableRow', {
  enumerable: true,
  get: function () {
    return convertTestResultsToTableRow;
  }
});
Object.defineProperty(exports, 'extractVariablesFromTemplate', {
  enumerable: true,
  get: function () {
    return extractVariablesFromTemplate;
  }
});
Object.defineProperty(exports, 'extractVariablesFromTemplates', {
  enumerable: true,
  get: function () {
    return extractVariablesFromTemplates;
  }
});
Object.defineProperty(exports, 'failApiCall', {
  enumerable: true,
  get: function () {
    return failApiCall;
  }
});
Object.defineProperty(exports, 'fetchCsvFromGoogleSheet', {
  enumerable: true,
  get: function () {
    return fetchCsvFromGoogleSheet;
  }
});
Object.defineProperty(exports, 'formatOpenAiError', {
  enumerable: true,
  get: function () {
    return formatOpenAiError;
  }
});
Object.defineProperty(exports, 'getNunjucksEngine', {
  enumerable: true,
  get: function () {
    return getNunjucksEngine;
  }
});
Object.defineProperty(exports, 'getNunjucksEngineForFilePath', {
  enumerable: true,
  get: function () {
    return getNunjucksEngineForFilePath;
  }
});
Object.defineProperty(exports, 'getResolvedRelativePath', {
  enumerable: true,
  get: function () {
    return getResolvedRelativePath;
  }
});
Object.defineProperty(exports, 'getTokenUsage', {
  enumerable: true,
  get: function () {
    return getTokenUsage;
  }
});
Object.defineProperty(exports, 'loadFunction', {
  enumerable: true,
  get: function () {
    return loadFunction;
  }
});
Object.defineProperty(exports, 'maybeLoadConfigFromExternalFile', {
  enumerable: true,
  get: function () {
    return maybeLoadConfigFromExternalFile;
  }
});
Object.defineProperty(exports, 'maybeLoadFromExternalFile', {
  enumerable: true,
  get: function () {
    return maybeLoadFromExternalFile;
  }
});
Object.defineProperty(exports, 'maybeLoadToolsFromExternalFile', {
  enumerable: true,
  get: function () {
    return maybeLoadToolsFromExternalFile;
  }
});
Object.defineProperty(exports, 'parseFileUrl', {
  enumerable: true,
  get: function () {
    return parseFileUrl;
  }
});
Object.defineProperty(exports, 'parsePathOrGlob', {
  enumerable: true,
  get: function () {
    return parsePathOrGlob;
  }
});
Object.defineProperty(exports, 'printBorder', {
  enumerable: true,
  get: function () {
    return printBorder;
  }
});
Object.defineProperty(exports, 'readFilters', {
  enumerable: true,
  get: function () {
    return readFilters;
  }
});
Object.defineProperty(exports, 'readOutput', {
  enumerable: true,
  get: function () {
    return readOutput;
  }
});
Object.defineProperty(exports, 'renderEnvOnlyInObject', {
  enumerable: true,
  get: function () {
    return renderEnvOnlyInObject;
  }
});
Object.defineProperty(exports, 'renderVarsInObject', {
  enumerable: true,
  get: function () {
    return renderVarsInObject;
  }
});
Object.defineProperty(exports, 'resultIsForTestCase', {
  enumerable: true,
  get: function () {
    return resultIsForTestCase;
  }
});
Object.defineProperty(exports, 'setupEnv', {
  enumerable: true,
  get: function () {
    return setupEnv;
  }
});
Object.defineProperty(exports, 'validateFunctionCall', {
  enumerable: true,
  get: function () {
    return validateFunctionCall;
  }
});
Object.defineProperty(exports, 'writeMultipleOutputs', {
  enumerable: true,
  get: function () {
    return writeMultipleOutputs;
  }
});
Object.defineProperty(exports, 'writeOutput', {
  enumerable: true,
  get: function () {
    return writeOutput;
  }
});
//# sourceMappingURL=util-CtKyTm4Y.cjs.map