"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.readAssertions = exports.runCompareAssertion = exports.isSql = exports.runAssertion = exports.runAssertions = exports.MODEL_GRADED_ASSERTION_TYPES = void 0;
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const node_util_1 = __importDefault(require("node:util"));
const async_1 = __importDefault(require("async"));
const rouge_1 = __importDefault(require("rouge"));
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
const js_yaml_1 = __importDefault(require("js-yaml"));
const ajv_1 = __importDefault(require("ajv"));
const ajv_formats_1 = __importDefault(require("ajv-formats"));
const rfdc_1 = __importDefault(require("rfdc"));
const fastest_levenshtein_1 = require("fastest-levenshtein");
const cliState_1 = __importDefault(require("./cliState"));
const telemetry_1 = __importDefault(require("./telemetry"));
const logger_1 = __importDefault(require("./logger"));
const fetch_1 = require("./fetch");
const util_1 = require("./util");
const matchers_1 = require("./matchers");
const openaiUtil_1 = require("./providers/openaiUtil");
const wrapper_1 = require("./python/wrapper");
const esm_1 = require("./esm");
const types_1 = require("./types");
const AssertionsResult_1 = require("./assertions/AssertionsResult");
const shared_1 = require("./providers/shared");
const ASSERTIONS_MAX_CONCURRENCY = process.env.PROMPTFOO_ASSERTIONS_MAX_CONCURRENCY
    ? parseInt(process.env.PROMPTFOO_ASSERTIONS_MAX_CONCURRENCY, 10)
    : 3;
exports.MODEL_GRADED_ASSERTION_TYPES = new Set([
    'answer-relevance',
    'context-faithfulness',
    'context-recall',
    'context-relevance',
    'llm-rubric',
    'model-graded-closedqa',
    'factuality',
    'model-graded-factuality',
]);
const ajv = new ajv_1.default();
(0, ajv_formats_1.default)(ajv);
const nunjucks = (0, util_1.getNunjucksEngine)();
const clone = (0, rfdc_1.default)();
function getFinalTest(test, assertion) {
    // Deep copy
    const ret = clone(test);
    // Assertion provider overrides test provider
    ret.options = ret.options || {};
    ret.options.provider = assertion.provider || ret.options.provider;
    ret.options.rubricPrompt = assertion.rubricPrompt || ret.options.rubricPrompt;
    return Object.freeze(ret);
}
function coerceString(value) {
    if (typeof value === 'string') {
        return value;
    }
    return JSON.stringify(value);
}
function handleRougeScore(baseType, assertion, expected, output, inverted) {
    const fnName = baseType[baseType.length - 1];
    const rougeMethod = rouge_1.default[fnName];
    const score = rougeMethod(output, expected);
    const pass = score >= (assertion.threshold || 0.75) != inverted;
    return {
        pass,
        score: inverted ? 1 - score : score,
        reason: pass
            ? `${baseType.toUpperCase()} score ${score.toFixed(2)} is greater than or equal to threshold ${assertion.threshold || 0.75}`
            : `${baseType.toUpperCase()} score ${score.toFixed(2)} is less than threshold ${assertion.threshold || 0.75}`,
        assertion,
    };
}
async function runAssertions({ prompt, provider, test, output, latencyMs, logProbs, cost, }) {
    if (!test.assert || test.assert.length < 1) {
        return AssertionsResult_1.AssertionsResult.noAssertsResult();
    }
    const mainAssertResult = new AssertionsResult_1.AssertionsResult({
        threshold: test.threshold,
    });
    const subAssertResults = [];
    const asserts = test.assert
        .map((assertion, i) => {
        if (assertion.type === 'assert-set') {
            const subAssertResult = new AssertionsResult_1.AssertionsResult({
                threshold: assertion.threshold,
                parentAssertionSet: {
                    assertionSet: assertion,
                    index: i,
                },
            });
            subAssertResults.push(subAssertResult);
            return assertion.assert.map((subAssert, j) => {
                return {
                    assertion: subAssert,
                    assertResult: subAssertResult,
                    index: j,
                };
            });
        }
        return { assertion, assertResult: mainAssertResult, index: i };
    })
        .flat();
    await async_1.default.forEachOfLimit(asserts, ASSERTIONS_MAX_CONCURRENCY, async ({ assertion, assertResult, index }) => {
        if (assertion.type.startsWith('select-')) {
            // Select-type assertions are handled separately because they depend on multiple outputs.
            return;
        }
        const result = await runAssertion({
            prompt,
            provider,
            assertion,
            test,
            output,
            latencyMs,
            logProbs,
            cost,
        });
        assertResult.addResult({
            index,
            result,
            metric: assertion.metric,
            weight: assertion.weight,
        });
    });
    subAssertResults.forEach((subAssertResult) => {
        const result = subAssertResult.testResult();
        const { index, assertionSet: { metric, weight }, } = subAssertResult.parentAssertionSet;
        mainAssertResult.addResult({
            index,
            result,
            metric,
            weight,
        });
    });
    return mainAssertResult.testResult();
}
exports.runAssertions = runAssertions;
async function runAssertion({ prompt, provider, assertion, test, output, latencyMs, logProbs, cost, }) {
    let pass = false;
    let score = 0.0;
    (0, tiny_invariant_1.default)(assertion.type, `Assertion must have a type: ${JSON.stringify(assertion)}`);
    const inverse = assertion.type.startsWith('not-');
    const baseType = inverse ? assertion.type.slice(4) : assertion.type;
    telemetry_1.default.record('assertion_used', {
        type: baseType,
    });
    if (assertion.transform) {
        output = await (0, util_1.transformOutput)(assertion.transform, output, {
            vars: test.vars,
            prompt: { label: prompt },
        });
    }
    const outputString = coerceString(output);
    const context = {
        prompt,
        vars: test.vars || {},
        test,
        logProbs,
    };
    // Render assertion values
    let renderedValue = assertion.value;
    let valueFromScript;
    if (typeof renderedValue === 'string') {
        if (renderedValue.startsWith('file://')) {
            const basePath = cliState_1.default.basePath || '';
            const filePath = path_1.default.resolve(basePath, renderedValue.slice('file://'.length));
            if (filePath.endsWith('.js') || filePath.endsWith('.cjs') || filePath.endsWith('.mjs')) {
                const requiredModule = await (0, esm_1.importModule)(filePath);
                if (typeof requiredModule === 'function') {
                    valueFromScript = await Promise.resolve(requiredModule(output, context));
                }
                else if (requiredModule.default && typeof requiredModule.default === 'function') {
                    valueFromScript = await Promise.resolve(requiredModule.default(output, context));
                }
                else {
                    throw new Error(`Assertion malformed: ${filePath} must export a function or have a default export as a function`);
                }
                logger_1.default.debug(`Javascript script ${filePath} output: ${valueFromScript}`);
            }
            else if (filePath.endsWith('.py')) {
                try {
                    const pythonScriptOutput = await (0, wrapper_1.runPython)(filePath, 'get_assert', [output, context]);
                    valueFromScript = pythonScriptOutput;
                    logger_1.default.debug(`Python script ${filePath} output: ${valueFromScript}`);
                }
                catch (error) {
                    return {
                        pass: false,
                        score: 0,
                        reason: error.message,
                        assertion,
                    };
                }
            }
            else if (filePath.endsWith('.json')) {
                renderedValue = JSON.parse(fs_1.default.readFileSync(path_1.default.resolve(basePath, filePath), 'utf8'));
            }
            else if (filePath.endsWith('.yaml') || filePath.endsWith('.yml')) {
                renderedValue = js_yaml_1.default.load(fs_1.default.readFileSync(path_1.default.resolve(basePath, filePath), 'utf8'));
            }
            else if (filePath.endsWith('.txt')) {
                // Trim to remove trailing newline
                renderedValue = fs_1.default.readFileSync(path_1.default.resolve(basePath, filePath), 'utf8').trim();
            }
            else {
                throw new Error(`Unsupported file type: ${filePath}`);
            }
        }
        else {
            // It's a normal string value
            renderedValue = nunjucks.renderString(renderedValue, test.vars || {});
        }
    }
    else if (renderedValue && Array.isArray(renderedValue)) {
        // Unpack the array
        renderedValue = renderedValue.map((v) => nunjucks.renderString(String(v), test.vars || {}));
    }
    // Transform test
    test = getFinalTest(test, assertion);
    if (baseType === 'equals') {
        if (typeof renderedValue === 'object') {
            pass = node_util_1.default.isDeepStrictEqual(renderedValue, JSON.parse(outputString)) !== inverse;
            renderedValue = JSON.stringify(renderedValue);
        }
        else {
            pass = (renderedValue == outputString) !== inverse;
        }
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass
                ? 'Assertion passed'
                : `Expected output "${renderedValue}" to ${inverse ? 'not ' : ''}equal "${outputString}"`,
            assertion,
        };
    }
    if (baseType === 'is-json') {
        let parsedJson;
        try {
            parsedJson = JSON.parse(outputString);
            pass = !inverse;
        }
        catch (err) {
            pass = inverse;
        }
        if (pass && renderedValue) {
            let validate;
            if (typeof renderedValue === 'string') {
                if (renderedValue.startsWith('file://')) {
                    // Reference the JSON schema from external file
                    const schema = valueFromScript;
                    (0, tiny_invariant_1.default)(schema, 'is-json references a file that does not export a JSON schema');
                    validate = ajv.compile(schema);
                }
                else {
                    const scheme = js_yaml_1.default.load(renderedValue);
                    validate = ajv.compile(scheme);
                }
            }
            else if (typeof renderedValue === 'object') {
                // Value is JSON schema
                validate = ajv.compile(renderedValue);
            }
            else {
                throw new Error('is-json assertion must have a string or object value');
            }
            pass = validate(parsedJson);
            if (!pass) {
                return {
                    pass,
                    score: 0,
                    reason: `JSON does not conform to the provided schema. Errors: ${ajv.errorsText(validate.errors)}`,
                    assertion,
                };
            }
        }
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass ? 'Assertion passed' : 'Expected output to be valid JSON',
            assertion,
        };
    }
    if (baseType === 'is-sql') {
        return isSql(outputString, renderedValue, inverse, assertion);
    }
    if (baseType === 'contains-sql') {
        const match = outputString.match(/```(?:sql)?([^`]+)```/);
        if (match) {
            const sqlCode = match[1].trim();
            return isSql(sqlCode, renderedValue, inverse, assertion);
        }
        else {
            return isSql(outputString, renderedValue, inverse, assertion);
        }
    }
    if (baseType === 'contains') {
        (0, tiny_invariant_1.default)(renderedValue, '"contains" assertion type must have a string or number value');
        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'number', '"contains" assertion type must have a string or number value');
        pass = outputString.includes(String(renderedValue)) !== inverse;
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass
                ? 'Assertion passed'
                : `Expected output to ${inverse ? 'not ' : ''}contain "${renderedValue}"`,
            assertion,
        };
    }
    if (baseType === 'contains-any') {
        (0, tiny_invariant_1.default)(renderedValue, '"contains-any" assertion type must have a value');
        if (typeof renderedValue === 'string') {
            renderedValue = renderedValue.split(',').map((v) => v.trim());
        }
        (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"contains-any" assertion type must have an array value');
        pass = renderedValue.some((value) => outputString.includes(String(value))) !== inverse;
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass
                ? 'Assertion passed'
                : `Expected output to ${inverse ? 'not ' : ''}contain one of "${renderedValue.join(', ')}"`,
            assertion,
        };
    }
    if (baseType === 'icontains-any') {
        (0, tiny_invariant_1.default)(renderedValue, '"icontains-any" assertion type must have a value');
        if (typeof renderedValue === 'string') {
            renderedValue = renderedValue.split(',').map((v) => v.trim());
        }
        (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"icontains-any" assertion type must have an array value');
        pass =
            renderedValue.some((value) => outputString.toLowerCase().includes(String(value).toLowerCase())) !== inverse;
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass
                ? 'Assertion passed'
                : `Expected output to ${inverse ? 'not ' : ''}contain one of "${renderedValue.join(', ')}"`,
            assertion,
        };
    }
    if (baseType === 'contains-all') {
        (0, tiny_invariant_1.default)(renderedValue, '"contains-all" assertion type must have a value');
        if (typeof renderedValue === 'string') {
            renderedValue = renderedValue.split(',').map((v) => v.trim());
        }
        (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"contains-all" assertion type must have an array value');
        pass = renderedValue.every((value) => outputString.includes(String(value))) !== inverse;
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass
                ? 'Assertion passed'
                : `Expected output to ${inverse ? 'not ' : ''}contain all of "${renderedValue.join(', ')}"`,
            assertion,
        };
    }
    if (baseType === 'icontains-all') {
        (0, tiny_invariant_1.default)(renderedValue, '"icontains-all" assertion type must have a value');
        if (typeof renderedValue === 'string') {
            renderedValue = renderedValue.split(',').map((v) => v.trim());
        }
        (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"icontains-all" assertion type must have an array value');
        pass =
            renderedValue.every((value) => outputString.toLowerCase().includes(String(value).toLowerCase())) !== inverse;
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass
                ? 'Assertion passed'
                : `Expected output to ${inverse ? 'not ' : ''}contain all of "${renderedValue.join(', ')}"`,
            assertion,
        };
    }
    if (baseType === 'regex') {
        (0, tiny_invariant_1.default)(renderedValue, '"regex" assertion type must have a string value');
        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"regex" assertion type must have a string value');
        const regex = new RegExp(renderedValue);
        pass = regex.test(outputString) !== inverse;
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass
                ? 'Assertion passed'
                : `Expected output to ${inverse ? 'not ' : ''}match regex "${renderedValue}"`,
            assertion,
        };
    }
    if (baseType === 'icontains') {
        (0, tiny_invariant_1.default)(renderedValue, '"icontains" assertion type must have a string or number value');
        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'number', '"icontains" assertion type must have a string or number value');
        pass = outputString.toLowerCase().includes(String(renderedValue).toLowerCase()) !== inverse;
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass
                ? 'Assertion passed'
                : `Expected output to ${inverse ? 'not ' : ''}contain "${renderedValue}"`,
            assertion,
        };
    }
    if (baseType === 'starts-with') {
        (0, tiny_invariant_1.default)(renderedValue, '"starts-with" assertion type must have a string value');
        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"starts-with" assertion type must have a string value');
        pass = outputString.startsWith(String(renderedValue)) !== inverse;
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass
                ? 'Assertion passed'
                : `Expected output to ${inverse ? 'not ' : ''}start with "${renderedValue}"`,
            assertion,
        };
    }
    if (baseType === 'contains-json') {
        let errorMessage = 'Expected output to contain valid JSON';
        const jsonOutputs = containsJSON(outputString);
        for (const jsonMatch of jsonOutputs) {
            pass = jsonMatch !== inverse;
            if (pass && renderedValue) {
                let validate;
                if (typeof renderedValue === 'string') {
                    if (renderedValue.startsWith('file://')) {
                        // Reference the JSON schema from external file
                        const schema = valueFromScript;
                        (0, tiny_invariant_1.default)(schema, 'contains-json references a file that does not export a JSON schema');
                        validate = ajv.compile(schema);
                    }
                    else {
                        const scheme = js_yaml_1.default.load(renderedValue);
                        validate = ajv.compile(scheme);
                    }
                }
                else if (typeof renderedValue === 'object') {
                    // Value is JSON schema
                    validate = ajv.compile(renderedValue);
                }
                else {
                    throw new Error('contains-json assertion must have a string or object value');
                }
                pass = validate(jsonMatch);
                if (pass) {
                    break;
                }
                else {
                    errorMessage = `JSON does not conform to the provided schema. Errors: ${ajv.errorsText(validate.errors)}`;
                }
            }
        }
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass ? 'Assertion passed' : errorMessage,
            assertion,
        };
    }
    if (baseType === 'is-valid-openai-tools-call') {
        const toolsOutput = output;
        if (!Array.isArray(toolsOutput) ||
            toolsOutput.length === 0 ||
            typeof toolsOutput[0].function.name !== 'string' ||
            typeof toolsOutput[0].function.arguments !== 'string') {
            return {
                pass: false,
                score: 0,
                reason: `OpenAI did not return a valid-looking tools response: ${JSON.stringify(toolsOutput)}`,
                assertion,
            };
        }
        try {
            toolsOutput.forEach((toolOutput) => (0, openaiUtil_1.validateFunctionCall)(toolOutput.function, provider.config.tools?.map((tool) => tool.function), test.vars));
            return {
                pass: true,
                score: 1,
                reason: 'Assertion passed',
                assertion,
            };
        }
        catch (err) {
            return {
                pass: false,
                score: 0,
                reason: err.message,
                assertion,
            };
        }
    }
    if (baseType === 'is-valid-openai-function-call') {
        const functionOutput = output;
        if (typeof functionOutput !== 'object' ||
            typeof functionOutput.name !== 'string' ||
            typeof functionOutput.arguments !== 'string') {
            return {
                pass: false,
                score: 0,
                reason: `OpenAI did not return a valid-looking function call: ${JSON.stringify(functionOutput)}`,
                assertion,
            };
        }
        try {
            (0, openaiUtil_1.validateFunctionCall)(functionOutput, provider.config.functions, test.vars);
            return {
                pass: true,
                score: 1,
                reason: 'Assertion passed',
                assertion,
            };
        }
        catch (err) {
            return {
                pass: false,
                score: 0,
                reason: err.message,
                assertion,
            };
        }
    }
    if (baseType === 'javascript') {
        try {
            const validateResult = async (result) => {
                result = await Promise.resolve(result);
                if (typeof result === 'boolean' || typeof result === 'number' || (0, types_1.isGradingResult)(result)) {
                    return result;
                }
                else {
                    throw new Error(`Custom function must return a boolean, number, or GradingResult object. Got type ${typeof result}: ${JSON.stringify(result)}`);
                }
            };
            if (typeof assertion.value === 'function') {
                let ret = assertion.value(outputString, context);
                ret = await validateResult(ret);
                if (!ret.assertion) {
                    // Populate the assertion object if the custom function didn't return it.
                    const functionString = assertion.value.toString();
                    ret.assertion = {
                        type: 'javascript',
                        value: functionString.length > 50 ? functionString.slice(0, 50) + '...' : functionString,
                    };
                }
                return ret;
            }
            (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'javascript assertion must have a string value');
            let result;
            if (typeof valueFromScript !== 'undefined') {
                (0, tiny_invariant_1.default)(typeof valueFromScript === 'boolean' ||
                    typeof valueFromScript === 'number' ||
                    typeof valueFromScript === 'object', `Javascript assertion script must return a boolean, number, or object (${assertion.value})`);
                result = await validateResult(valueFromScript);
            }
            else {
                const functionBody = renderedValue.includes('\n')
                    ? renderedValue
                    : `return ${renderedValue}`;
                const customFunction = new Function('output', 'context', functionBody);
                result = await validateResult(customFunction(output, context));
            }
            if (typeof result === 'boolean') {
                pass = result !== inverse;
                score = pass ? 1 : 0;
            }
            else if (typeof result === 'number') {
                pass = assertion.threshold ? result >= assertion.threshold : result > 0;
                score = result;
            }
            else if (typeof result === 'object') {
                return result;
            }
            else {
                throw new Error('Custom function must return a boolean or number');
            }
        }
        catch (err) {
            return {
                pass: false,
                score: 0,
                reason: `Custom function threw error: ${err.message}
Stack Trace: ${err.stack}
${renderedValue}`,
                assertion,
            };
        }
        return {
            pass,
            score,
            reason: pass
                ? 'Assertion passed'
                : `Custom function returned ${inverse ? 'true' : 'false'}
${renderedValue}`,
            assertion,
        };
    }
    if (baseType === 'python') {
        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'python assertion must have a string value');
        try {
            let result;
            if (typeof valueFromScript !== 'undefined') {
                result = valueFromScript;
            }
            else {
                const isMultiline = renderedValue.includes('\n');
                let indentStyle = '    ';
                if (isMultiline) {
                    // Detect the indentation style of the first indented line
                    const match = renderedValue.match(/^(?!\s*$)\s+/m);
                    if (match) {
                        indentStyle = match[0];
                    }
                }
                const pythonScript = `import json

def main(output, context):
${isMultiline
                    ? renderedValue
                        .split('\n')
                        .map((line) => `${indentStyle}${line}`)
                        .join('\n')
                    : `    return ${renderedValue}`}
`;
                result = await (0, wrapper_1.runPythonCode)(pythonScript, 'main', [output, context]);
            }
            if ((typeof result === 'boolean' && result) ||
                (typeof result === 'string' && result.toLowerCase() === 'true')) {
                pass = true;
                score = 1.0;
            }
            else if ((typeof result === 'boolean' && !result) ||
                (typeof result === 'string' && result.toLowerCase() === 'false')) {
                pass = false;
                score = 0.0;
            }
            else if (typeof result === 'string' && result.startsWith('{')) {
                let parsed;
                try {
                    parsed = JSON.parse(result);
                }
                catch (err) {
                    throw new Error(`Invalid JSON: ${err} when parsing result: ${result}`);
                }
                if (!(0, types_1.isGradingResult)(parsed)) {
                    throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Got instead: ${result}`);
                }
                return parsed;
            }
            else if (typeof result === 'object') {
                if (!(0, types_1.isGradingResult)(result)) {
                    throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Got instead:\n${JSON.stringify(result, null, 2)}`);
                }
                const pythonGradingResult = result;
                if (assertion.threshold && pythonGradingResult.score < assertion.threshold) {
                    pythonGradingResult.pass = false;
                    pythonGradingResult.reason = `Python score ${pythonGradingResult.score} is less than threshold ${assertion.threshold}`;
                }
                return {
                    ...pythonGradingResult,
                    assertion,
                };
            }
            else {
                score = parseFloat(String(result));
                pass = assertion.threshold ? score >= assertion.threshold : score > 0;
                if (isNaN(score)) {
                    throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Instead got:\n${result}`);
                }
                if (typeof assertion.threshold !== 'undefined' && score < assertion.threshold) {
                    pass = false;
                }
            }
        }
        catch (err) {
            return {
                pass: false,
                score: 0,
                reason: `Python code execution failed: ${err.message}`,
                assertion,
            };
        }
        return {
            pass,
            score,
            reason: pass
                ? 'Assertion passed'
                : `Python code returned ${pass ? 'true' : 'false'}\n${assertion.value}`,
            assertion,
        };
    }
    if (baseType === 'similar') {
        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || Array.isArray(renderedValue), 'Similarity assertion type must have a string or array of strings value');
        if (Array.isArray(renderedValue)) {
            let minScore = Infinity;
            for (const value of renderedValue) {
                const result = await (0, matchers_1.matchesSimilarity)(value, outputString, assertion.threshold || 0.75, inverse, test.options);
                if (result.pass) {
                    return {
                        assertion,
                        ...result,
                    };
                }
                if (result.score < minScore) {
                    minScore = result.score;
                }
            }
            return {
                assertion,
                pass: false,
                score: minScore,
                reason: `None of the provided values met the similarity threshold`,
            };
        }
        else {
            return {
                assertion,
                ...(await (0, matchers_1.matchesSimilarity)(renderedValue, outputString, assertion.threshold || 0.75, inverse, test.options)),
            };
        }
    }
    if (baseType === 'llm-rubric') {
        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'undefined', '"llm-rubric" assertion type must have a string value');
        if (test.options?.rubricPrompt) {
            if (typeof test.options.rubricPrompt === 'object') {
                test.options.rubricPrompt = JSON.stringify(test.options.rubricPrompt);
            }
        }
        // Update the assertion value. This allows the web view to display the prompt.
        assertion.value = assertion.value || test.options?.rubricPrompt;
        return {
            assertion,
            ...(await (0, matchers_1.matchesLlmRubric)(renderedValue || '', outputString, test.options, test.vars)),
        };
    }
    if (baseType === 'model-graded-factuality' || baseType === 'factuality') {
        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'factuality assertion type must have a string value');
        (0, tiny_invariant_1.default)(prompt, 'factuality assertion type must have a prompt');
        if (test.options?.rubricPrompt) {
            // Substitute vars in prompt
            (0, tiny_invariant_1.default)(typeof test.options.rubricPrompt === 'string', 'rubricPrompt must be a string');
            test.options.rubricPrompt = nunjucks.renderString(test.options.rubricPrompt, test.vars || {});
        }
        return {
            assertion,
            ...(await (0, matchers_1.matchesFactuality)(prompt, renderedValue, outputString, test.options, test.vars)),
        };
    }
    if (baseType === 'model-graded-closedqa') {
        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'model-graded-closedqa assertion type must have a string value');
        (0, tiny_invariant_1.default)(prompt, 'model-graded-closedqa assertion type must have a prompt');
        if (test.options?.rubricPrompt) {
            // Substitute vars in prompt
            (0, tiny_invariant_1.default)(typeof test.options.rubricPrompt === 'string', 'rubricPrompt must be a string');
            test.options.rubricPrompt = nunjucks.renderString(test.options.rubricPrompt, test.vars || {});
        }
        return {
            assertion,
            ...(await (0, matchers_1.matchesClosedQa)(prompt, renderedValue, outputString, test.options, test.vars)),
        };
    }
    if (baseType === 'answer-relevance') {
        (0, tiny_invariant_1.default)(typeof output === 'string', 'answer-relevance assertion type must evaluate a string output');
        (0, tiny_invariant_1.default)(prompt, 'answer-relevance assertion type must have a prompt');
        const input = typeof test.vars?.query === 'string' ? test.vars.query : prompt;
        return {
            assertion,
            ...(await (0, matchers_1.matchesAnswerRelevance)(input, output, assertion.threshold || 0, test.options)),
        };
    }
    if (baseType === 'context-recall') {
        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'context-recall assertion type must have a string value');
        (0, tiny_invariant_1.default)(prompt, 'context-recall assertion type must have a prompt');
        return {
            assertion,
            ...(await (0, matchers_1.matchesContextRecall)(typeof test.vars?.context === 'string' ? test.vars.context : prompt, renderedValue, assertion.threshold || 0, test.options, test.vars)),
        };
    }
    if (baseType === 'context-relevance') {
        (0, tiny_invariant_1.default)(test.vars, 'context-relevance assertion type must have a vars object');
        (0, tiny_invariant_1.default)(typeof test.vars.query === 'string', 'context-relevance assertion type must have a question var');
        (0, tiny_invariant_1.default)(typeof test.vars.context === 'string', 'context-relevance assertion type must have a context var');
        return {
            assertion,
            ...(await (0, matchers_1.matchesContextRelevance)(test.vars.query, test.vars.context, assertion.threshold || 0, test.options)),
        };
    }
    if (baseType === 'context-faithfulness') {
        (0, tiny_invariant_1.default)(test.vars, 'context-faithfulness assertion type must have a vars object');
        (0, tiny_invariant_1.default)(typeof test.vars.query === 'string', 'context-faithfulness assertion type must have a question var');
        (0, tiny_invariant_1.default)(typeof test.vars.context === 'string', 'context-faithfulness assertion type must have a context var');
        (0, tiny_invariant_1.default)(typeof output === 'string', 'context-faithfulness assertion type must have a string output');
        return {
            assertion,
            ...(await (0, matchers_1.matchesContextFaithfulness)(test.vars.query, output, test.vars.context, assertion.threshold || 0, test.options)),
        };
    }
    if (baseType === 'moderation') {
        (0, tiny_invariant_1.default)(prompt, 'moderation assertion type must have a prompt');
        (0, tiny_invariant_1.default)(typeof output === 'string', 'moderation assertion type must have a string output');
        (0, tiny_invariant_1.default)(!assertion.value ||
            (Array.isArray(assertion.value) && typeof assertion.value[0] === 'string'), 'moderation assertion value must be a string array if set');
        if (prompt[0] === '[' || prompt[0] === '{') {
            // Try to extract the last user message from OpenAI-style prompts.
            try {
                const parsedPrompt = (0, shared_1.parseChatPrompt)(prompt, null);
                if (parsedPrompt && parsedPrompt.length > 0) {
                    prompt = parsedPrompt[parsedPrompt.length - 1].content;
                }
            }
            catch (err) {
                // Ignore error
            }
        }
        const moderationResult = await (0, matchers_1.matchesModeration)({
            userPrompt: prompt,
            assistantResponse: output,
            categories: Array.isArray(assertion.value) ? assertion.value : [],
        }, test.options);
        pass = moderationResult.pass;
        return {
            pass,
            score: moderationResult.score,
            reason: moderationResult.reason,
            assertion,
        };
    }
    if (baseType === 'webhook') {
        (0, tiny_invariant_1.default)(renderedValue, '"webhook" assertion type must have a URL value');
        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"webhook" assertion type must have a URL value');
        try {
            const context = {
                prompt,
                vars: test.vars || {},
            };
            const response = await (0, fetch_1.fetchWithRetries)(renderedValue, {
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                },
                body: JSON.stringify({ output, context }),
            }, process.env.WEBHOOK_TIMEOUT ? parseInt(process.env.WEBHOOK_TIMEOUT, 10) : 5000);
            if (!response.ok) {
                throw new Error(`Webhook response status: ${response.status}`);
            }
            const jsonResponse = await response.json();
            pass = jsonResponse.pass !== inverse;
            score =
                typeof jsonResponse.score === 'undefined'
                    ? pass
                        ? 1
                        : 0
                    : inverse
                        ? 1 - jsonResponse.score
                        : jsonResponse.score;
            const reason = jsonResponse.reason ||
                (pass ? 'Assertion passed' : `Webhook returned ${inverse ? 'true' : 'false'}`);
            return {
                pass,
                score,
                reason,
                assertion,
            };
        }
        catch (err) {
            return {
                pass: false,
                score: 0,
                reason: `Webhook error: ${err.message}`,
                assertion,
            };
        }
    }
    if (baseType === 'rouge-n') {
        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"rouge" assertion type must be a string value');
        return handleRougeScore(baseType, assertion, renderedValue, outputString, inverse);
    }
    if (baseType === 'levenshtein') {
        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"levenshtein" assertion type must have a string value');
        const levDistance = (0, fastest_levenshtein_1.distance)(outputString, renderedValue);
        pass = levDistance <= (assertion.threshold || 5);
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass
                ? 'Assertion passed'
                : `Levenshtein distance ${levDistance} is greater than threshold ${assertion.threshold || 5}`,
            assertion,
        };
    }
    if (baseType === 'classifier') {
        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'undefined', '"classifier" assertion type must have a string value or be undefined');
        // Assertion provider overrides test provider
        test.options = test.options || {};
        test.options.provider = assertion.provider || test.options.provider;
        const classificationResult = await (0, matchers_1.matchesClassification)(renderedValue, outputString, assertion.threshold ?? 1, test.options);
        if (inverse) {
            classificationResult.pass = !classificationResult.pass;
            classificationResult.score = 1 - classificationResult.score;
        }
        return {
            assertion,
            ...classificationResult,
        };
    }
    if (baseType === 'latency') {
        if (!assertion.threshold) {
            throw new Error('Latency assertion must have a threshold in milliseconds');
        }
        if (!latencyMs) {
            throw new Error('Latency assertion does not support cached results. Rerun the eval with --no-cache');
        }
        pass = latencyMs <= assertion.threshold;
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass
                ? 'Assertion passed'
                : `Latency ${latencyMs}ms is greater than threshold ${assertion.threshold}ms`,
            assertion,
        };
    }
    if (baseType === 'perplexity') {
        if (!logProbs || logProbs.length === 0) {
            throw new Error('Perplexity assertion does not support providers that do not return logProbs');
        }
        const sumLogProbs = logProbs.reduce((acc, logProb) => acc + logProb, 0);
        const avgLogProb = sumLogProbs / logProbs.length;
        const perplexity = Math.exp(-avgLogProb);
        pass = assertion.threshold ? perplexity <= assertion.threshold : true;
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass
                ? 'Assertion passed'
                : `Perplexity ${perplexity.toFixed(2)} is greater than threshold ${assertion.threshold}`,
            assertion,
        };
    }
    if (baseType === 'perplexity-score') {
        if (!logProbs || logProbs.length === 0) {
            throw new Error('perplexity-score assertion does not support providers that do not return logProbs');
        }
        const sumLogProbs = logProbs.reduce((acc, logProb) => acc + logProb, 0);
        const avgLogProb = sumLogProbs / logProbs.length;
        const perplexity = Math.exp(-avgLogProb);
        const perplexityNorm = 1 / (1 + perplexity);
        pass = assertion.threshold ? perplexityNorm >= assertion.threshold : true;
        return {
            pass,
            score: perplexityNorm,
            reason: pass
                ? 'Assertion passed'
                : `Perplexity score ${perplexityNorm.toFixed(2)} is less than threshold ${assertion.threshold}`,
            assertion,
        };
    }
    if (baseType === 'cost') {
        if (!assertion.threshold) {
            throw new Error('Cost assertion must have a threshold');
        }
        if (typeof cost === 'undefined') {
            throw new Error('Cost assertion does not support providers that do not return cost');
        }
        pass = cost <= assertion.threshold;
        return {
            pass,
            score: pass ? 1 : 0,
            reason: pass
                ? 'Assertion passed'
                : `Cost ${cost.toPrecision(2)} is greater than threshold ${assertion.threshold}`,
            assertion,
        };
    }
    throw new Error('Unknown assertion type: ' + assertion.type);
}
exports.runAssertion = runAssertion;
function containsJSON(str) {
    // This will extract all json objects from a string
    const jsonObjects = [];
    let openBracket = str.indexOf('{');
    let closeBracket = str.indexOf('}', openBracket);
    // Iterate over the string until we find a valid JSON-like pattern
    // Iterate over all trailing } until the contents parse as json
    while (openBracket !== -1) {
        const jsonStr = str.slice(openBracket, closeBracket + 1);
        try {
            jsonObjects.push(JSON.parse(jsonStr));
            // This is a valid JSON object, so start looking for
            // an opening bracket after the last closing bracket
            openBracket = str.indexOf('{', closeBracket + 1);
            closeBracket = str.indexOf('}', openBracket);
        }
        catch (err) {
            // Not a valid object, move on to the next closing bracket
            closeBracket = str.indexOf('}', closeBracket + 1);
            while (closeBracket === -1) {
                // No closing brackets made a valid json object, so
                // start looking with the next opening bracket
                openBracket = str.indexOf('{', openBracket + 1);
                closeBracket = str.indexOf('}', openBracket);
            }
        }
    }
    return jsonObjects;
}
async function isSql(outputString, renderedValue, inverse, assertion) {
    let pass = false;
    let parsedSql;
    let databaseType = 'MySQL';
    let whiteTableList;
    let whiteColumnList;
    if (renderedValue && typeof renderedValue === 'object') {
        const value = renderedValue;
        databaseType = value.database || 'MySQL';
        whiteTableList = value.allowedTables;
        whiteColumnList = value.allowedColumns;
    }
    if (renderedValue && typeof renderedValue !== 'object') {
        throw new Error('is-sql assertion must have a object value.');
    }
    const { Parser: SqlParser } = await Promise.resolve().then(() => __importStar(require('node-sql-parser'))).catch(() => {
        throw new Error('node-sql-parser is not installed. Please install it first');
    });
    const sqlParser = new SqlParser();
    const opt = { database: databaseType };
    const failureReasons = [];
    try {
        parsedSql = sqlParser.astify(outputString, opt);
        pass = !inverse;
    }
    catch (err) {
        pass = inverse;
        failureReasons.push(`SQL statement does not conform to the provided ${databaseType} database syntax.`);
    }
    if (whiteTableList) {
        opt.type = 'table';
        try {
            sqlParser.whiteListCheck(outputString, whiteTableList, opt);
        }
        catch (err) {
            pass = inverse;
            const error = err;
            failureReasons.push(`SQL validation failed: ${error.message}.`);
        }
    }
    if (whiteColumnList) {
        opt.type = 'column';
        try {
            sqlParser.whiteListCheck(outputString, whiteColumnList, opt);
        }
        catch (err) {
            pass = inverse;
            const error = err;
            failureReasons.push(`SQL validation failed: ${error.message}.`);
        }
    }
    if (inverse && pass === false && failureReasons.length === 0) {
        failureReasons.push('The output SQL statement is valid');
    }
    return {
        pass,
        score: pass ? 1 : 0,
        reason: pass ? 'Assertion passed' : failureReasons.join(' '),
        assertion,
    };
}
exports.isSql = isSql;
async function runCompareAssertion(test, assertion, outputs) {
    (0, tiny_invariant_1.default)(typeof assertion.value === 'string', 'select-best must have a string value');
    test.options = test.options || {};
    test.options.provider = assertion.provider || test.options.provider;
    test.options.rubricPrompt = assertion.rubricPrompt || test.options.rubricPrompt;
    const comparisonResults = await (0, matchers_1.matchesSelectBest)(assertion.value, outputs, test.options, test.vars);
    return comparisonResults.map((result) => ({
        ...result,
        assertion,
    }));
}
exports.runCompareAssertion = runCompareAssertion;
async function readAssertions(filePath) {
    try {
        const assertions = js_yaml_1.default.load(fs_1.default.readFileSync(filePath, 'utf-8'));
        if (!Array.isArray(assertions) || assertions[0]?.type === undefined) {
            throw new Error('Assertions file must be an array of assertion objects');
        }
        return assertions;
    }
    catch (err) {
        throw new Error(`Failed to read assertions from ${filePath}:\n${err}`);
    }
}
exports.readAssertions = readAssertions;
// These exports are used by the node.js package (index.ts)
exports.default = {
    matchesSimilarity: matchers_1.matchesSimilarity,
    matchesClassification: matchers_1.matchesClassification,
    matchesLlmRubric: matchers_1.matchesLlmRubric,
    matchesFactuality: matchers_1.matchesFactuality,
    matchesClosedQa: matchers_1.matchesClosedQa,
    matchesAnswerRelevance: matchers_1.matchesAnswerRelevance,
    matchesContextRecall: matchers_1.matchesContextRecall,
    matchesContextRelevance: matchers_1.matchesContextRelevance,
    matchesContextFaithfulness: matchers_1.matchesContextFaithfulness,
    matchesComparisonBoolean: matchers_1.matchesSelectBest,
    matchesModeration: matchers_1.matchesModeration,
};
//# sourceMappingURL=assertions.js.map