#!/usr/bin/env node
import { k as cliState_default, o as logger_default, x as getEnvBool } from "./logger-CjS9Mbik.js";
import { t as fetchWithProxy, v as TERMINAL_MAX_WIDTH, w as VERSION } from "./fetch-Brl8CXgW.js";
import { t as invariant } from "./invariant-CzGQ4hYY.js";
import { a as OutputFileExtension, d as isApiProvider, f as isProviderOptions, o as ResultFailureReason } from "./types-atpq0XCc.js";
import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-DgUMlmAt.js";
import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-CNEnghzm.js";
import { r as runPython } from "./pythonUtils-Dxv9iWbf.js";
import { t as convertTestResultsToTableRow } from "./exportToFile-B1s42t-e.js";
import dotenv from "dotenv";
import * as fs$2 from "fs";
import * as path$1 from "path";
import path from "path";
import * as os$1 from "os";
import yaml from "js-yaml";
import * as fsPromises from "fs/promises";
import dedent from "dedent";
import deepEqual from "fast-deep-equal";
import { parse as parse$1 } from "csv-parse/sync";
import { globSync, hasMagic } from "glob";
import nunjucks from "nunjucks";
import { stringify } from "csv-stringify/sync";
import { XMLBuilder } from "fast-xml-parser";

//#region src/util/provider.ts
function canonicalizeProviderId(id) {
	if (id.startsWith("file://")) {
		const filePath = id.slice(7);
		return path$1.isAbsolute(filePath) ? id : `file://${path$1.resolve(filePath)}`;
	}
	for (const prefix of [
		"exec:",
		"python:",
		"golang:"
	]) if (id.startsWith(prefix)) {
		const filePath = id.slice(prefix.length);
		if (filePath.includes("/") || filePath.includes("\\")) return `${prefix}${path$1.resolve(filePath)}`;
		return id;
	}
	if ((id.endsWith(".js") || id.endsWith(".ts") || id.endsWith(".mjs")) && (id.includes("/") || id.includes("\\"))) return `file://${path$1.resolve(id)}`;
	return id;
}
function getProviderLabel(provider) {
	return provider?.label && typeof provider.label === "string" ? provider.label : void 0;
}
function providerToIdentifier(provider) {
	if (!provider) return;
	if (typeof provider === "string") return canonicalizeProviderId(provider);
	const label = getProviderLabel(provider);
	if (label) return label;
	if (isApiProvider(provider)) return canonicalizeProviderId(provider.id());
	if (isProviderOptions(provider)) {
		if (provider.id) return canonicalizeProviderId(provider.id);
		return;
	}
	if (typeof provider === "object" && "id" in provider && typeof provider.id === "string") return canonicalizeProviderId(provider.id);
}

//#endregion
//#region src/util/comparison.ts
/**
* Explicit runtime variable names that don't follow the underscore convention.
* These are added during evaluation but aren't part of the original test definition.
*
* - sessionId: Added by multi-turn strategy providers (GOAT, Crescendo)
*
* Note: Variables starting with underscore (e.g., _conversation) are automatically
* treated as runtime variables and filtered out.
*/
const EXPLICIT_RUNTIME_VAR_KEYS = ["sessionId"];
/**
* Checks if a variable key is a runtime-only variable that should be filtered
* when comparing test cases.
*
* Runtime variables are identified by:
* 1. Starting with underscore (_) - convention for internal/runtime vars
* 2. Being in the explicit runtime var list (for legacy vars like sessionId)
*/
function isRuntimeVar(key) {
	return key.startsWith("_") || EXPLICIT_RUNTIME_VAR_KEYS.includes(key);
}
/**
* Filters out runtime-only variables that are added during evaluation
* but aren't part of the original test definition.
*
* This is used when comparing test cases to determine if a result
* corresponds to a particular test, regardless of runtime state.
*
* Runtime variables are identified by:
* - Starting with underscore (e.g., _conversation, _metadata)
* - Being in the explicit list (e.g., sessionId for backward compatibility)
*/
function filterRuntimeVars(vars) {
	if (!vars) return vars;
	const filtered = {};
	for (const [key, value] of Object.entries(vars)) if (!isRuntimeVar(key)) filtered[key] = value;
	return filtered;
}
function varsMatch(vars1, vars2) {
	return deepEqual(vars1, vars2);
}
/**
* Generate a unique key for a test case for deduplication purposes.
* Excludes runtime variables and includes strategyId to distinguish tests
* with the same prompt but different strategies.
*
* @param testCase - The test case to generate a key for
* @returns A JSON string that uniquely identifies the test case
*/
function getTestCaseDeduplicationKey(testCase) {
	const filteredVars = filterRuntimeVars(testCase.vars);
	const strategyId = testCase.metadata?.strategyId || "none";
	return JSON.stringify({
		vars: filteredVars,
		strategyId
	});
}
function resultIsForTestCase(result, testCase) {
	const providersMatch = testCase.provider ? providerToIdentifier(testCase.provider) === providerToIdentifier(result.provider) : true;
	const resultVars = filterRuntimeVars(result.vars);
	return varsMatch(filterRuntimeVars(testCase.vars), resultVars) && providersMatch;
}

//#endregion
//#region src/util/env.ts
/**
* Load environment variables from .env file(s).
* @param envPath - Single path, array of paths, or undefined for default .env loading.
*                  When paths are explicitly specified, all files must exist or an error is thrown.
*                  When multiple files are provided, later files override values from earlier files.
*/
function setupEnv(envPath) {
	if (envPath) {
		const paths = (Array.isArray(envPath) ? envPath : [envPath]).flatMap((p) => p.includes(",") ? p.split(",").map((s) => s.trim()) : p.trim()).filter((p) => p.length > 0);
		if (paths.length === 0) {
			dotenv.config({ quiet: true });
			return;
		}
		for (const p of paths) if (!fs$2.existsSync(p)) throw new Error(`Environment file not found: ${p}`);
		if (paths.length === 1) logger_default.info(`Loading environment variables from ${paths[0]}`);
		else logger_default.info(`Loading environment variables from: ${paths.join(", ")}`);
		const pathArg = paths.length === 1 ? paths[0] : paths;
		dotenv.config({
			path: pathArg,
			override: true,
			quiet: true
		});
	} else dotenv.config({ quiet: true });
}

//#endregion
//#region src/util/functions/loadFunction.ts
const functionCache = {};
/**
* Loads a function from a JavaScript or Python file
* @param options Options for loading the function
* @returns The loaded function
*/
async function loadFunction({ filePath, functionName, defaultFunctionName = "func", basePath = cliState_default.basePath, useCache = true }) {
	const cacheKey = `${filePath}${functionName ? `:${functionName}` : ""}`;
	if (useCache && functionCache[cacheKey]) return functionCache[cacheKey];
	const resolvedPath = basePath ? path.resolve(basePath, filePath) : filePath;
	if (!isJavascriptFile(resolvedPath) && !resolvedPath.endsWith(".py")) throw new Error(`File must be a JavaScript (${JAVASCRIPT_EXTENSIONS.join(", ")}) or Python (.py) file`);
	try {
		let func;
		if (isJavascriptFile(resolvedPath)) {
			const module = await importModule(resolvedPath, functionName);
			let moduleFunc;
			if (functionName) moduleFunc = module;
			else moduleFunc = typeof module === "function" ? module : module?.default?.default || module?.default || module?.[defaultFunctionName] || module;
			if (typeof moduleFunc !== "function") throw new Error(functionName ? `JavaScript file must export a "${functionName}" function` : `JavaScript file must export a function (as default export or named export "${defaultFunctionName}")`);
			func = moduleFunc;
		} else {
			const result = (...args) => runPython(resolvedPath, functionName || defaultFunctionName, args);
			func = result;
		}
		if (useCache) functionCache[cacheKey] = func;
		return func;
	} catch (err) {
		logger_default.error(`Failed to load function: ${err.message}`);
		throw err;
	}
}
/**
* Extracts the file path and function name from a file:// URL
* @param fileUrl The file:// URL (e.g., "file://path/to/file.js:functionName")
* @returns The file path and optional function name
*/
function parseFileUrl(fileUrl) {
	if (!fileUrl.startsWith("file://")) throw new Error("URL must start with file://");
	const urlWithoutProtocol = fileUrl.slice(7);
	const lastColonIndex = urlWithoutProtocol.lastIndexOf(":");
	if (lastColonIndex > 1) return {
		filePath: urlWithoutProtocol.slice(0, lastColonIndex),
		functionName: urlWithoutProtocol.slice(lastColonIndex + 1)
	};
	return { filePath: urlWithoutProtocol };
}

//#endregion
//#region src/util/templates.ts
/**
* Get a Nunjucks engine instance with optional filters and configuration.
* @param filters - Optional map of custom Nunjucks filters.
* @param throwOnUndefined - Whether to throw an error on undefined variables.
* @param isGrader - Whether this engine is being used in a grader context.
* Nunjucks is always enabled in grader mode.
* @returns A configured Nunjucks environment.
*/
function getNunjucksEngine(filters, throwOnUndefined = false, isGrader = false) {
	if (!isGrader && getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return { renderString: (template) => template };
	const env = nunjucks.configure({
		autoescape: false,
		throwOnUndefined
	});
	const envGlobals = {
		...getEnvBool("PROMPTFOO_DISABLE_TEMPLATE_ENV_VARS", getEnvBool("PROMPTFOO_SELF_HOSTED", false)) ? {} : process.env,
		...cliState_default.config?.env
	};
	env.addGlobal("env", envGlobals);
	env.addFilter("load", function(str) {
		return JSON.parse(str);
	});
	if (filters) for (const [name, filter] of Object.entries(filters)) env.addFilter(name, filter);
	return env;
}
/**
* Parse Nunjucks template to extract variables.
* @param template - The Nunjucks template string.
* @returns An array of variables used in the template.
*/
function extractVariablesFromTemplate(template) {
	const variableSet = /* @__PURE__ */ new Set();
	const regex = /\{\{[\s]*([^{}\s|]+)[\s]*(?:\|[^}]+)?\}\}|\{%[\s]*(?:if|for)[\s]+([^{}\s]+)[\s]*.*?%\}/g;
	template = template.replace(/\{#[\s\S]*?#\}/g, "");
	let match;
	while ((match = regex.exec(template)) !== null) {
		const variable = match[1] || match[2];
		if (variable) variableSet.add(variable);
	}
	const forLoopRegex = /\{%[\s]*for[\s]+(\w+)[\s]+in[\s]+(\w+)[\s]*%\}/g;
	while ((match = forLoopRegex.exec(template)) !== null) {
		variableSet.delete(match[1]);
		variableSet.add(match[2]);
	}
	return Array.from(variableSet);
}
/**
* Extract variables from multiple Nunjucks templates.
* @param templates - An array of Nunjucks template strings.
* @returns An array of variables used in the templates.
*/
function extractVariablesFromTemplates(templates) {
	const variableSet = /* @__PURE__ */ new Set();
	for (const template of templates) extractVariablesFromTemplate(template).forEach((variable) => variableSet.add(variable));
	return Array.from(variableSet);
}

//#endregion
//#region src/util/render.ts
/**
* Renders ONLY environment variable templates in an object, leaving all other templates untouched.
* This allows env vars to be resolved at provider load time while preserving runtime var templates.
*
* Supports full Nunjucks syntax for env vars including filters and expressions:
* - {{ env.VAR_NAME }}
* - {{ env['VAR-NAME'] }}
* - {{ env["VAR-NAME"] }}
* - {{ env.VAR | default('fallback') }}
* - {{ env.VAR | upper }}
*
* Preserves non-env templates for runtime rendering:
* - {{ vars.x }} - preserved as literal
* - {{ prompt }} - preserved as literal
*
* Implementation: Uses regex to find env templates, delegates to Nunjucks for rendering.
* This ensures full Nunjucks feature support while preserving non-env templates.
*
* @param obj - The object to process
* @returns The object with only env templates rendered
*/
function renderEnvOnlyInObject(obj, envOverrides) {
	if (getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return obj;
	if (typeof obj === "string") {
		if (obj.length > 5e4) {
			logger_default.warn(`String too long (${obj.length} chars) for template matching. Skipping env var rendering.`);
			return obj;
		}
		const nunjucks$1 = getNunjucksEngine();
		const baseEnvGlobals = nunjucks$1.getGlobal("env");
		const envGlobals = envOverrides ? {
			...baseEnvGlobals,
			...envOverrides
		} : baseEnvGlobals;
		return obj.replace(/\{\{(?:[^}]|\}(?!\}))*\}\}/g, (match) => {
			if (!match.match(/\benv\.|env\[/)) return match;
			const varMatch = match.match(/env\.(\w+)|env\[['"]([^'"]+)['"]\]/);
			const varName = varMatch?.[1] || varMatch?.[2];
			if (match.includes("|") || varName && varName in envGlobals) try {
				return nunjucks$1.renderString(match, { env: envGlobals });
			} catch (_error) {
				return match;
			}
			return match;
		});
	}
	if (Array.isArray(obj)) return obj.map((item) => renderEnvOnlyInObject(item, envOverrides));
	if (typeof obj === "object" && obj !== null) {
		const result = {};
		for (const key in obj) result[key] = renderEnvOnlyInObject(obj[key], envOverrides);
		return result;
	}
	return obj;
}
function renderVarsInObject(obj, vars) {
	if (!vars || getEnvBool("PROMPTFOO_DISABLE_TEMPLATING")) return obj;
	if (typeof obj === "string") return getNunjucksEngine().renderString(obj, vars);
	if (Array.isArray(obj)) return obj.map((item) => renderVarsInObject(item, vars));
	if (typeof obj === "object" && obj !== null) {
		const result = {};
		for (const key in obj) result[key] = renderVarsInObject(obj[key], vars);
		return result;
	} else if (typeof obj === "function") return renderVarsInObject(obj({ vars }));
	return obj;
}

//#endregion
//#region src/util/file.ts
/**
* Simple Nunjucks engine specifically for file paths
* This function is separate from the main getNunjucksEngine to avoid circular dependencies
*/
function getNunjucksEngineForFilePath() {
	const env = nunjucks.configure({ autoescape: false });
	env.addGlobal("env", {
		...process.env,
		...cliState_default.config?.env
	});
	return env;
}
/**
* Loads content from an external file if the input is a file path, otherwise
* returns the input as-is. Supports Nunjucks templating for file paths.
*
* @param filePath - The input to process. Can be a file path string starting with "file://",
* an array of file paths, or any other type of data.
* @param context - Optional context to control file loading behavior. 'assertion' context
* preserves Python/JS file references instead of loading their content.
* @returns The loaded content if the input was a file path, otherwise the original input.
* For JSON and YAML files, the content is parsed into an object.
* For other file types, the raw file content is returned as a string.
*
* @throws {Error} If the specified file does not exist.
*/
function maybeLoadFromExternalFile(filePath, context) {
	if (Array.isArray(filePath)) return filePath.map((path$2) => {
		return maybeLoadFromExternalFile(path$2, context);
	});
	if (typeof filePath !== "string") return filePath;
	if (!filePath.startsWith("file://")) return filePath;
	const renderedFilePath = getNunjucksEngineForFilePath().renderString(filePath, {});
	const { filePath: cleanPath, functionName } = parseFileUrl(renderedFilePath);
	if (context === "assertion" && (cleanPath.endsWith(".py") || isJavascriptFile(cleanPath))) {
		logger_default.debug(`Preserving Python/JS file reference in assertion context: ${renderedFilePath}`);
		return renderedFilePath;
	}
	if (context === "vars") {
		logger_default.debug(`Preserving file reference in vars context: ${renderedFilePath}`);
		return renderedFilePath;
	}
	if (functionName && (cleanPath.endsWith(".py") || isJavascriptFile(cleanPath))) return renderedFilePath;
	const pathToUse = functionName && !(cleanPath.endsWith(".py") || isJavascriptFile(cleanPath)) ? renderedFilePath.slice(7) : cleanPath;
	const resolvedPath = path$1.resolve(cliState_default.basePath || "", pathToUse);
	if (hasMagic(pathToUse)) {
		const matchedFiles = globSync(resolvedPath, { windowsPathsNoEscape: true });
		if (matchedFiles.length === 0) throw new Error(`No files found matching pattern: ${resolvedPath}`);
		const allContents = [];
		for (const matchedFile of matchedFiles) {
			let contents$1;
			try {
				contents$1 = fs$2.readFileSync(matchedFile, "utf8");
			} catch (error) {
				if (error.code === "ENOENT") {
					logger_default.debug(`File disappeared during glob expansion: ${matchedFile}`);
					continue;
				}
				throw error;
			}
			if (matchedFile.endsWith(".json")) {
				const parsed = JSON.parse(contents$1);
				if (Array.isArray(parsed)) allContents.push(...parsed);
				else allContents.push(parsed);
			} else if (matchedFile.endsWith(".yaml") || matchedFile.endsWith(".yml")) {
				const parsed = yaml.load(contents$1);
				if (parsed === null || parsed === void 0) continue;
				if (Array.isArray(parsed)) allContents.push(...parsed);
				else allContents.push(parsed);
			} else if (matchedFile.endsWith(".csv")) {
				const records = parse$1(contents$1, { columns: true });
				if (records.length > 0 && Object.keys(records[0]).length === 1) allContents.push(...records.map((record) => Object.values(record)[0]));
				else allContents.push(...records);
			} else allContents.push(contents$1);
		}
		return allContents;
	}
	const finalPath = resolvedPath;
	let contents;
	try {
		contents = fs$2.readFileSync(finalPath, "utf8");
	} catch (error) {
		if (error.code === "ENOENT") throw new Error(`File does not exist: ${finalPath}`);
		throw new Error(`Failed to read file ${finalPath}: ${error}`);
	}
	if (finalPath.endsWith(".json")) try {
		return JSON.parse(contents);
	} catch (error) {
		throw new Error(`Failed to parse JSON file ${finalPath}: ${error}`);
	}
	if (finalPath.endsWith(".yaml") || finalPath.endsWith(".yml")) try {
		return yaml.load(contents);
	} catch (error) {
		throw new Error(`Failed to parse YAML file ${finalPath}: ${error}`);
	}
	if (finalPath.endsWith(".csv")) {
		const records = parse$1(contents, { columns: true });
		if (records.length > 0 && Object.keys(records[0]).length === 1) return records.map((record) => Object.values(record)[0]);
		return records;
	}
	return contents;
}
/**
* Resolves a relative file path with respect to a base path, handling cloud configuration appropriately.
* When using a cloud configuration, the current working directory is always used instead of the context's base path.
*
* @param filePath - The relative or absolute file path to resolve.
* @param isCloudConfig - Whether this is a cloud configuration.
* @returns The resolved absolute file path.
*/
function getResolvedRelativePath(filePath, isCloudConfig) {
	if (path$1.isAbsolute(filePath) || !isCloudConfig) return filePath;
	return path$1.join(process.cwd(), filePath);
}
/**
* Recursively loads external file references from a configuration object.
*
* @param config - The configuration object to process
* @param context - Optional context to control file loading behavior
* @returns The configuration with external file references resolved
*/
function maybeLoadConfigFromExternalFile(config, context) {
	if (Array.isArray(config)) return config.map((item) => maybeLoadConfigFromExternalFile(item, context));
	if (config && typeof config === "object" && config !== null) {
		const result = {};
		for (const key of Object.keys(config)) {
			const childContext = key === "value" && typeof config === "object" && config && "type" in config && typeof config.type === "string" && (config.type === "python" || config.type === "javascript") ? "assertion" : key === "vars" ? "vars" : context;
			result[key] = maybeLoadConfigFromExternalFile(config[key], childContext);
		}
		return result;
	}
	return maybeLoadFromExternalFile(config, context);
}
/**
* Parses a file path or glob pattern to extract function names and file extensions.
* Function names can be specified in the filename like this:
* prompt.py:myFunction or prompts.js:myFunction.
* @param basePath - The base path for file resolution.
* @param promptPath - The path or glob pattern.
* @returns Parsed details including function name, file extension, and directory status.
*/
function parsePathOrGlob(basePath, promptPath) {
	if (promptPath.startsWith("file://")) promptPath = promptPath.slice(7);
	const filePath = path$1.resolve(basePath, promptPath);
	let filename = path$1.relative(basePath, filePath);
	let functionName;
	if (filename.includes(":")) {
		const lastColonIndex = filename.lastIndexOf(":");
		if (lastColonIndex > 1) {
			const pathWithoutFunction = filename.slice(0, lastColonIndex);
			if (isJavascriptFile(pathWithoutFunction) || pathWithoutFunction.endsWith(".py") || pathWithoutFunction.endsWith(".go") || pathWithoutFunction.endsWith(".rb")) {
				functionName = filename.slice(lastColonIndex + 1);
				filename = pathWithoutFunction;
			}
		}
	}
	let stats;
	try {
		stats = fs$2.statSync(path$1.join(basePath, filename));
	} catch (err) {
		if (getEnvBool("PROMPTFOO_STRICT_FILES")) throw err;
	}
	const normalizedFilePath = filePath.replace(/\\/g, "/");
	const isPathPattern = stats?.isDirectory() || hasMagic(promptPath) || hasMagic(normalizedFilePath);
	const safeFilename = path$1.relative(basePath, safeResolve(basePath, filename));
	return {
		extension: isPathPattern ? void 0 : path$1.parse(safeFilename).ext,
		filePath: path$1.join(basePath, safeFilename),
		functionName,
		isPathPattern
	};
}
function readOutput(outputPath) {
	const ext = path$1.parse(outputPath).ext.slice(1);
	switch (ext) {
		case "json": return JSON.parse(fs$2.readFileSync(outputPath, "utf-8"));
		default: throw new Error(`Unsupported output file format: ${ext} currently only supports json`);
	}
}
/**
* Load custom Nunjucks filters from external files.
* Note: If a glob pattern matches multiple files, only the last file's export is used.
* Each filter name should typically resolve to a single file.
*/
async function readFilters(filters, basePath = "") {
	const ret = {};
	for (const [name, filterPath] of Object.entries(filters)) {
		const filePaths = globSync(path$1.join(basePath, filterPath), { windowsPathsNoEscape: true });
		for (const filePath of filePaths) ret[name] = await importModule(path$1.resolve(filePath));
	}
	return ret;
}
/**
* Renders variables in a tools object and loads from external file if applicable.
* This function combines renderVarsInObject and maybeLoadFromExternalFile into a single step
* specifically for handling tools configurations.
*
* Supports loading from JSON, YAML, Python, and JavaScript files.
*
* @param tools - The tools configuration object or array to process.
* @param vars - Variables to use for rendering.
* @returns The processed tools configuration with variables rendered and content loaded from files if needed.
* @throws {Error} If the loaded tools are in an invalid format
*/
async function maybeLoadToolsFromExternalFile(tools, vars) {
	const rendered = renderVarsInObject(tools, vars);
	if (typeof rendered === "string" && rendered.startsWith("file://")) {
		const { filePath, functionName } = parseFileUrl(rendered);
		if (functionName && (filePath.endsWith(".py") || isJavascriptFile(filePath))) {
			const fileType = filePath.endsWith(".py") ? "Python" : "JavaScript";
			logger_default.debug(`[maybeLoadToolsFromExternalFile] Loading tools from ${fileType} file: ${filePath}:${functionName}`);
			try {
				let toolDefinitions;
				if (filePath.endsWith(".py")) {
					const absPath = safeResolve(cliState_default.basePath || process.cwd(), filePath);
					logger_default.debug(`[maybeLoadToolsFromExternalFile] Resolved Python path: ${absPath}`);
					toolDefinitions = await runPython(absPath, functionName, []);
				} else {
					const absPath = safeResolve(cliState_default.basePath || process.cwd(), filePath);
					logger_default.debug(`[maybeLoadToolsFromExternalFile] Resolved JavaScript path: ${absPath}`);
					const module = await importModule(absPath);
					const fn = module[functionName] || module.default?.[functionName];
					if (typeof fn !== "function") {
						const availableExports = Object.keys(module).filter((k) => k !== "default");
						const basePath = cliState_default.basePath || process.cwd();
						throw new Error(`Function "${functionName}" not found in ${filePath}. Available exports: ${availableExports.length > 0 ? availableExports.join(", ") : "(none)"}\nResolved from: ${basePath}`);
					}
					toolDefinitions = await Promise.resolve(fn());
				}
				if (!toolDefinitions || typeof toolDefinitions === "string" || typeof toolDefinitions === "number" || typeof toolDefinitions === "boolean") throw new Error(`Function "${functionName}" must return an array or object of tool definitions, but returned: ${toolDefinitions === null ? "null" : typeof toolDefinitions}`);
				logger_default.debug(`[maybeLoadToolsFromExternalFile] Successfully loaded ${Array.isArray(toolDefinitions) ? toolDefinitions.length : "object"} tools`);
				return toolDefinitions;
			} catch (err) {
				const errorMessage = err instanceof Error ? err.message : String(err);
				const basePath = cliState_default.basePath || process.cwd();
				throw new Error(`Failed to load tools from ${rendered}:\n${errorMessage}\n\nMake sure the function "${functionName}" exists and returns a valid tool definition array.\nResolved from: ${basePath}`);
			}
		}
		if (filePath.endsWith(".py") || isJavascriptFile(filePath)) {
			const ext = filePath.endsWith(".py") ? "Python" : "JavaScript";
			const basePath = cliState_default.basePath || process.cwd();
			throw new Error(`Cannot load tools from ${rendered}\n${ext} files require a function name. Use this format:\n  tools: file://${filePath}:get_tools\n\nYour ${ext} file should export a function that returns tool definitions:\n` + (filePath.endsWith(".py") ? `  def get_tools():\n      return [{"type": "function", "function": {...}}]` : `  module.exports.get_tools = () => [{ type: "function", function: {...} }];`) + `\n\nResolved from: ${basePath}`);
		}
	}
	if (Array.isArray(rendered)) {
		const results = await Promise.all(rendered.map((item) => maybeLoadToolsFromExternalFile(item, vars)));
		if (results.every((r) => Array.isArray(r))) return results.flat();
		return results;
	}
	if (typeof rendered !== "string") return rendered;
	const loaded = maybeLoadFromExternalFile(rendered);
	if (loaded !== void 0 && loaded !== null && typeof loaded === "string") {
		if (loaded.startsWith("file://")) throw new Error(`Failed to load tools from ${loaded}\nEnsure the file exists and contains valid JSON or YAML tool definitions.`);
		if (loaded.includes("def ") || loaded.includes("import ")) throw new Error("Invalid tools configuration: file appears to contain Python code.\nPython files require a function name. Use this format:\n  tools: file://tools.py:get_tools");
		throw new Error("Invalid tools configuration: expected an array or object, but got a string.\nIf using file://, ensure the file contains valid JSON or YAML tool definitions.");
	}
	return loaded;
}

//#endregion
//#region src/googleSheets.ts
async function checkGoogleSheetAccess(url) {
	try {
		const response = await fetchWithProxy(url);
		if (response.ok) return {
			public: true,
			status: response.status
		};
		else return {
			public: false,
			status: response.status
		};
	} catch (error) {
		logger_default.error(`Error checking sheet access: ${error}`);
		return { public: false };
	}
}
async function fetchCsvFromGoogleSheetUnauthenticated(url) {
	const { parse: parseCsv } = await import("csv-parse/sync");
	const gid = new URL(url).searchParams.get("gid");
	const response = await fetchWithProxy(`${url.replace(/\/edit.*$/, "/export")}?format=csv${gid ? `&gid=${gid}` : ""}`);
	if (response.status !== 200) throw new Error(`Failed to fetch CSV from Google Sheets URL: ${url}`);
	return parseCsv(await response.text(), { columns: true });
}
async function fetchCsvFromGoogleSheetAuthenticated(url) {
	const { sheets: googleSheets, auth: googleAuth } = await import("@googleapis/sheets");
	const auth = new googleAuth.GoogleAuth({ scopes: ["https://www.googleapis.com/auth/spreadsheets.readonly"] });
	const sheets = googleSheets("v4");
	const match = url.match(/\/d\/([^/]+)/);
	if (!match) throw new Error(`Invalid Google Sheets URL: ${url}`);
	const spreadsheetId = match[1];
	let range;
	const gid = Number(new URL(url).searchParams.get("gid"));
	if (gid) {
		const sheet = (await sheets.spreadsheets.get({
			spreadsheetId,
			auth
		})).data.sheets?.find((sheet$1) => sheet$1.properties?.sheetId === gid);
		if (!sheet || !sheet.properties?.title) throw new Error(`Sheet not found for gid: ${gid}`);
		range = sheet.properties.title;
	} else {
		const firstSheet = (await sheets.spreadsheets.get({
			spreadsheetId,
			auth
		})).data.sheets?.[0];
		if (!firstSheet || !firstSheet.properties?.title) throw new Error(`No sheets found in spreadsheet`);
		range = firstSheet.properties.title;
	}
	const rows = (await sheets.spreadsheets.values.get({
		spreadsheetId,
		range,
		auth
	})).data.values;
	if (!rows?.length) throw new Error(`No data found in Google Sheets URL: ${url}`);
	const headers = rows[0];
	return rows.slice(1).map((row) => {
		const csvRow = {};
		headers.forEach((header, index) => {
			csvRow[header] = row[index] ?? "";
		});
		return csvRow;
	});
}
async function fetchCsvFromGoogleSheet(url) {
	const { public: isPublic } = await checkGoogleSheetAccess(url);
	logger_default.debug(`Google Sheets URL: ${url}, isPublic: ${isPublic}`);
	if (isPublic) return fetchCsvFromGoogleSheetUnauthenticated(url);
	return fetchCsvFromGoogleSheetAuthenticated(url);
}
async function writeCsvToGoogleSheet(rows, url) {
	const { sheets: googleSheets, auth: googleAuth } = await import("@googleapis/sheets");
	const auth = new googleAuth.GoogleAuth({ scopes: ["https://www.googleapis.com/auth/spreadsheets"] });
	const sheets = googleSheets("v4");
	const match = url.match(/\/d\/([^/]+)/);
	if (!match) throw new Error(`Invalid Google Sheets URL: ${url}`);
	const spreadsheetId = match[1];
	const headers = Object.keys(rows[0]);
	const values = [headers, ...rows.map((row) => headers.map((header) => row[header]))];
	const getColumnLetter = (col) => {
		let letter = "";
		while (col > 0) {
			col--;
			letter = String.fromCharCode(65 + col % 26) + letter;
			col = Math.floor(col / 26);
		}
		return letter;
	};
	const numRows = values.length;
	const numCols = headers.length;
	const endColumn = getColumnLetter(numCols);
	let range;
	const gid = Number(new URL(url).searchParams.get("gid"));
	if (gid) {
		const sheet = (await sheets.spreadsheets.get({
			spreadsheetId,
			auth
		})).data.sheets?.find((sheet$1) => sheet$1.properties?.sheetId === gid);
		if (!sheet || !sheet.properties?.title) throw new Error(`Sheet not found for gid: ${gid}`);
		range = `${sheet.properties.title}!A1:${endColumn}${numRows}`;
	} else {
		const newSheetTitle = `Sheet${Date.now()}`;
		await sheets.spreadsheets.batchUpdate({
			spreadsheetId,
			auth,
			requestBody: { requests: [{ addSheet: { properties: { title: newSheetTitle } } }] }
		});
		range = `${newSheetTitle}!A1:${endColumn}${numRows}`;
	}
	logger_default.debug(`Writing CSV to Google Sheets URL: ${url} with ${values.length} rows`);
	await sheets.spreadsheets.values.update({
		spreadsheetId,
		range,
		valueInputOption: "USER_ENTERED",
		auth,
		requestBody: { values }
	});
}

//#endregion
//#region src/util/exportToFile/getHeaderForTable.ts
function getHeaderForTable(eval_) {
	const varsForHeader = /* @__PURE__ */ new Set();
	if (typeof eval_.config.defaultTest === "object" && eval_.config.defaultTest?.vars) for (const varName of Object.keys(eval_.config.defaultTest.vars || {})) varsForHeader.add(varName);
	for (const result of eval_.results) if (result.testCase?.vars) for (const varName of Object.keys(result.testCase.vars)) varsForHeader.add(varName);
	const tests = eval_.config.tests;
	let testsArray = [];
	if (Array.isArray(tests)) testsArray = tests;
	else if (tests) testsArray = [tests];
	for (const test of testsArray) {
		if (typeof test === "string") continue;
		if (test && typeof test === "object" && "path" in test) continue;
		if (test && "vars" in test) for (const varName of Object.keys(test.vars || {})) varsForHeader.add(varName);
	}
	for (const scenario of eval_.config.scenarios || []) {
		if (typeof scenario === "string") continue;
		for (const config of scenario.config || []) for (const varName of Object.keys(config.vars || {})) varsForHeader.add(varName);
		for (const test of scenario.tests || []) {
			if (typeof test === "string") continue;
			for (const varName of Object.keys(test.vars || {})) varsForHeader.add(varName);
		}
	}
	return {
		vars: [...varsForHeader].sort(),
		prompts: eval_.prompts
	};
}

//#endregion
//#region src/util/output.ts
const outputToSimpleString = (output) => {
	const passFailText = output.pass ? "[PASS]" : output.failureReason === ResultFailureReason.ASSERT ? "[FAIL]" : "[ERROR]";
	const namedScoresText = Object.entries(output.namedScores).map(([name, value]) => `${name}: ${value?.toFixed(2)}`).join(", ");
	const scoreText = namedScoresText.length > 0 ? `(${output.score?.toFixed(2)}, ${namedScoresText})` : `(${output.score?.toFixed(2)})`;
	const gradingResultText = output.gradingResult ? `${output.pass ? "Pass" : "Fail"} Reason: ${output.gradingResult.reason}` : "";
	return dedent`
      ${passFailText} ${scoreText}

      ${output.text}

      ${gradingResultText}
    `.trim();
};
function createOutputMetadata(evalRecord) {
	let evaluationCreatedAt;
	if (evalRecord.createdAt) try {
		const date = new Date(evalRecord.createdAt);
		evaluationCreatedAt = Number.isNaN(date.getTime()) ? void 0 : date.toISOString();
	} catch {
		evaluationCreatedAt = void 0;
	}
	return {
		promptfooVersion: VERSION,
		nodeVersion: process.version,
		platform: os$1.platform(),
		arch: os$1.arch(),
		exportedAt: (/* @__PURE__ */ new Date()).toISOString(),
		evaluationCreatedAt,
		author: evalRecord.author
	};
}
/**
* JSON writer with improved error handling for large datasets.
* Provides helpful error messages when memory limits are exceeded.
*/
async function writeJsonOutputSafely(outputPath, evalRecord, shareableUrl) {
	const metadata = createOutputMetadata(evalRecord);
	try {
		const summary = await evalRecord.toEvaluateSummary();
		const outputData = {
			evalId: evalRecord.id,
			results: summary,
			config: evalRecord.config,
			shareableUrl,
			metadata
		};
		const jsonString = JSON.stringify(outputData, null, 2);
		await fsPromises.writeFile(outputPath, jsonString);
	} catch (error) {
		const msg = error?.message ?? "";
		const isStringLen = error instanceof RangeError && msg.includes("Invalid string length");
		const isHeapOOM = /heap out of memory|Array buffer allocation failed|ERR_STRING_TOO_LONG/i.test(msg);
		if (isStringLen || isHeapOOM) {
			const resultCount = await evalRecord.getResultsCount();
			logger_default.error(`Dataset too large for JSON export (${resultCount} results).`);
			throw new Error(`Dataset too large for JSON export. The evaluation has ${resultCount} results which exceeds memory limits. Consider using JSONL format instead: --output output.jsonl`);
		} else throw error;
	}
}
async function writeOutput(outputPath, evalRecord, shareableUrl) {
	if (outputPath.match(/^https:\/\/docs\.google\.com\/spreadsheets\//)) {
		const table = await evalRecord.getTable();
		invariant(table, "Table is required");
		const rows = table.body.map((row) => {
			const csvRow = {};
			table.head.vars.forEach((varName, index) => {
				csvRow[varName] = row.vars[index];
			});
			table.head.prompts.forEach((prompt, index) => {
				csvRow[`[${prompt.provider}] ${prompt.label}`] = outputToSimpleString(row.outputs[index]);
			});
			return csvRow;
		});
		logger_default.info(`Writing ${rows.length} rows to Google Sheets...`);
		await writeCsvToGoogleSheet(rows, outputPath);
		return;
	}
	const { data: outputExtension } = OutputFileExtension.safeParse(path$1.extname(outputPath).slice(1).toLowerCase());
	invariant(outputExtension, `Unsupported output file format ${outputExtension}. Please use one of: ${OutputFileExtension.options.join(", ")}.`);
	const outputDir = path$1.dirname(outputPath);
	await fsPromises.mkdir(outputDir, { recursive: true });
	const metadata = createOutputMetadata(evalRecord);
	if (outputExtension === "csv") {
		const headers = getHeaderForTable(evalRecord);
		const headerCsv = stringify([[...headers.vars, ...headers.prompts.map((prompt) => `[${prompt.provider}] ${prompt.label}`)]]);
		await fsPromises.writeFile(outputPath, headerCsv);
		for await (const batchResults of evalRecord.fetchResultsBatched()) {
			const tableRows = {};
			for (const result of batchResults) {
				if (!(result.testIdx in tableRows)) tableRows[result.testIdx] = [];
				tableRows[result.testIdx].push(result);
			}
			const batchCsv = stringify(Object.values(tableRows).map((results) => {
				const row = convertTestResultsToTableRow(results, headers.vars);
				return [...row.vars, ...row.outputs.map(outputToSimpleString)];
			}));
			await fsPromises.appendFile(outputPath, batchCsv);
		}
	} else if (outputExtension === "json") await writeJsonOutputSafely(outputPath, evalRecord, shareableUrl);
	else if (outputExtension === "yaml" || outputExtension === "yml" || outputExtension === "txt") {
		const summary = await evalRecord.toEvaluateSummary();
		await fsPromises.writeFile(outputPath, yaml.dump({
			evalId: evalRecord.id,
			results: summary,
			config: evalRecord.config,
			shareableUrl,
			metadata
		}));
	} else if (outputExtension === "html") {
		const table = await evalRecord.getTable();
		invariant(table, "Table is required");
		const summary = await evalRecord.toEvaluateSummary();
		const template = await fsPromises.readFile(path$1.join(getDirectory(), "tableOutput.html"), "utf-8");
		const htmlTable = [[...table.head.vars, ...table.head.prompts.map((prompt) => `[${prompt.provider}] ${prompt.label}`)], ...table.body.map((row) => [...row.vars, ...row.outputs.map(outputToSimpleString)])];
		const htmlOutput = getNunjucksEngine().renderString(template, {
			config: evalRecord.config,
			table: htmlTable,
			results: summary
		});
		await fsPromises.writeFile(outputPath, htmlOutput);
	} else if (outputExtension === "jsonl") {
		await fsPromises.writeFile(outputPath, "");
		for await (const batchResults of evalRecord.fetchResultsBatched()) {
			const text = batchResults.map((result) => JSON.stringify(result)).join(os$1.EOL) + os$1.EOL;
			await fsPromises.appendFile(outputPath, text);
		}
	} else if (outputExtension === "xml") {
		const summary = await evalRecord.toEvaluateSummary();
		const sanitizeForXml = (obj) => {
			if (obj === null || obj === void 0) return "";
			if (typeof obj === "boolean" || typeof obj === "number") return String(obj);
			if (typeof obj === "string") return obj;
			if (Array.isArray(obj)) return obj.map(sanitizeForXml);
			if (typeof obj === "object") {
				const sanitized = {};
				for (const [key, value] of Object.entries(obj)) sanitized[key] = sanitizeForXml(value);
				return sanitized;
			}
			return String(obj);
		};
		const xmlData = new XMLBuilder({
			ignoreAttributes: false,
			format: true,
			indentBy: "  "
		}).build({ promptfoo: {
			evalId: evalRecord.id,
			results: sanitizeForXml(summary),
			config: sanitizeForXml(evalRecord.config),
			shareableUrl: shareableUrl || ""
		} });
		await fsPromises.writeFile(outputPath, xmlData);
	}
}
async function writeMultipleOutputs(outputPaths, evalRecord, shareableUrl) {
	await Promise.all(outputPaths.map((outputPath) => writeOutput(outputPath, evalRecord, shareableUrl)));
}

//#endregion
//#region src/util/runtime.ts
function printBorder() {
	const border = "=".repeat(TERMINAL_MAX_WIDTH);
	logger_default.info(border);
}

//#endregion
export { resultIsForTestCase as C, getTestCaseDeduplicationKey as S, extractVariablesFromTemplates as _, fetchCsvFromGoogleSheet as a, parseFileUrl as b, maybeLoadConfigFromExternalFile as c, parsePathOrGlob as d, readFilters as f, extractVariablesFromTemplate as g, renderVarsInObject as h, writeOutput as i, maybeLoadFromExternalFile as l, renderEnvOnlyInObject as m, createOutputMetadata as n, getNunjucksEngineForFilePath as o, readOutput as p, writeMultipleOutputs as r, getResolvedRelativePath as s, printBorder as t, maybeLoadToolsFromExternalFile as u, getNunjucksEngine as v, setupEnv as x, loadFunction as y };
//# sourceMappingURL=util-BuhznZrR.js.map