"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.MODEL_GRADED_ASSERTION_TYPES = void 0;
exports.isApiCallError = isApiCallError;
exports.runAssertion = runAssertion;
exports.runAssertions = runAssertions;
exports.runCompareAssertion = runCompareAssertion;
exports.readAssertions = readAssertions;
const async_1 = __importDefault(require("async"));
const fs_1 = __importDefault(require("fs"));
const js_yaml_1 = __importDefault(require("js-yaml"));
const path_1 = __importDefault(require("path"));
const cliState_1 = __importDefault(require("../cliState"));
const envars_1 = require("../envars");
const logger_1 = __importDefault(require("../logger"));
const matchers_1 = require("../matchers");
const packageParser_1 = require("../providers/packageParser");
const pythonUtils_1 = require("../python/pythonUtils");
const types_1 = require("../types");
const fileExtensions_1 = require("../util/fileExtensions");
const invariant_1 = __importDefault(require("../util/invariant"));
const templates_1 = require("../util/templates");
const transform_1 = require("../util/transform");
const taskCompleteness_1 = require("./agentic/taskCompleteness");
const toolCorrectness_1 = require("./agentic/toolCorrectness");
const answerRelevance_1 = require("./answerRelevance");
const assertionsResult_1 = require("./assertionsResult");
const backgroundNoise_1 = require("./audio/backgroundNoise");
const callCategory_1 = require("./audio/callCategory");
const callDisconnection_1 = require("./audio/callDisconnection");
const callOutcome_1 = require("./audio/callOutcome");
const clarity_1 = require("./audio/clarity");
const combinedAudio_1 = require("./audio/combinedAudio");
const escalation_1 = require("./audio/escalation");
const falseIntentIdentification_1 = require("./audio/falseIntentIdentification");
const general_1 = require("./audio/general");
const greeting_1 = require("./audio/greeting");
const incompleteAgentResponse_1 = require("./audio/incompleteAgentResponse");
const interruption_1 = require("./audio/interruption");
const language_1 = require("./audio/language");
const languageSwitch_1 = require("./audio/languageSwitch");
const noAgentResponse_1 = require("./audio/noAgentResponse");
const repetition_1 = require("./audio/repetition");
const responseTime_1 = require("./audio/responseTime");
const sentiment_1 = require("./audio/sentiment");
const sttAcccuracy_1 = require("./audio/sttAcccuracy");
const sttCorrectness_1 = require("./audio/sttCorrectness");
const sttMissing_1 = require("./audio/sttMissing");
const sttNumberCorrectness_1 = require("./audio/sttNumberCorrectness");
const talkToListenRatio_1 = require("./audio/talkToListenRatio");
const bleu_1 = require("./bleu");
const classifier_1 = require("./classifier");
const codeExecuation_1 = require("./codeExecuation");
const contains_1 = require("./contains");
const contextFaithfulness_1 = require("./contextFaithfulness");
const contextRecall_1 = require("./contextRecall");
const contextRelevance_1 = require("./contextRelevance");
const conversationCompletenessAssertion_1 = require("./conversationCompletenessAssertion");
const conversationRelevancyAssertion_1 = require("./conversationRelevancyAssertion");
const conversationalGEval_1 = require("./conversationalGEval");
const cost_1 = require("./cost");
const equals_1 = require("./equals");
const factuality_1 = require("./factuality");
const functionToolCall_1 = require("./functionToolCall");
const geval_1 = require("./geval");
const gleu_1 = require("./gleu");
const guardrails_1 = require("./guardrails");
const hallucination_1 = require("./hallucination");
const handleCodeLanguage_1 = require("./handleCodeLanguage");
const handleMultimodalContextualPrecision_1 = require("./handleMultimodalContextualPrecision");
const handleMultimodalContextualRecall_1 = require("./handleMultimodalContextualRecall");
const handleMultimodalContextualRelevancy_1 = require("./handleMultimodalContextualRelevancy");
const handleMultimodalFaithfulness_1 = require("./handleMultimodalFaithfulness");
const gEval_1 = require("./multimodal/gEval");
const handleRoleAdherence_1 = require("./handleRoleAdherence");
const javascript_1 = require("./javascript");
const json_1 = require("./json");
const knowledgeRetention_1 = require("./knowledgeRetention");
const latency_1 = require("./latency");
const levenshtein_1 = require("./levenshtein");
const llmRubric_1 = require("./llmRubric");
const modelGradedClosedQa_1 = require("./modelGradedClosedQa");
const moderation_1 = require("./moderation");
const not_json_1 = require("./not-json");
const not_rouge_1 = require("./not-rouge");
const openai_1 = require("./openai");
const perplexity_1 = require("./perplexity");
const pi_1 = require("./pi");
const python_1 = require("./python");
const redteam_1 = require("./redteam");
const refusal_1 = require("./refusal");
const regex_1 = require("./regex");
const rouge_1 = require("./rouge");
const similar_1 = require("./similar");
const sql_1 = require("./sql");
const startsWith_1 = require("./startsWith");
const utils_1 = require("./utils");
const webhook_1 = require("./webhook");
const xml_1 = require("./xml");
const argumentCorrectness_1 = require("./agentic/argumentCorrectness");
const planCorrectness_1 = require("./agentic/planCorrectness");
const planAdherence_1 = require("./agentic/planAdherence");
const goalFulfillment_1 = require("./agentic/goalFulfillment");
const executionEfficiency_1 = require("./agentic/executionEfficiency");
const reasoningCoherence_1 = require("./agentic/reasoningCoherence");
const agentCustomEval_1 = require("./agentic/agentCustomEval");
const toolSelectionQuality_1 = require("./agentic/toolSelectionQuality");
const multimodalCommon_1 = require("./multimodal/multimodalCommon");
const visualQuality_1 = require("./video/visualQuality");
const textToVideo_1 = require("./video/textToVideo");
const factualConsistency_1 = require("./video/factualConsistency");
const temporalConsistency_1 = require("./video/temporalConsistency");
const gEval_2 = require("./video/gEval");
const imageEditing_1 = require("./multimodal/imageEditing");
const ASSERTIONS_MAX_CONCURRENCY = (0, envars_1.getEnvInt)('TRUSYS_ASSERTIONS_MAX_CONCURRENCY', 3);
exports.MODEL_GRADED_ASSERTION_TYPES = new Set([
    'answer-relevance',
    'context-faithfulness',
    'context-recall',
    'context-relevance',
    'factuality',
    'llm-rubric',
    'model-graded-closedqa',
    'model-graded-factuality',
    'plan-quality',
    'plan-adherence',
    'goal-fulfillment',
    'execution-efficiency',
    'reasoning-coherence',
    'agent-custom-eval',
    'tool-selection-quality',
]);
const nunjucks = (0, templates_1.getNunjucksEngine)();
/**
 * Checks if a string contains API call error patterns
 * @param message - The error message or reason string to check
 * @returns true if the message indicates an API call error
 */
function isApiCallError(message) {
    if (!message) {
        return false;
    }
    return (message.includes('API call error') ||
        message.includes('Request failed after') ||
        message.includes('retries'));
}
async function runAssertion({ prompt, provider, assertion, test, latencyMs, providerResponse, assertIndex, workspaceId, }) {
    const { cost, logProbs, output: originalOutput } = providerResponse;
    let output = originalOutput;
    (0, invariant_1.default)(assertion.type, `Assertion must have a type: ${JSON.stringify(assertion)}`);
    const inverse = assertion.type.startsWith('not-');
    if (assertion.type.startsWith('promptfoo:redteam:')) {
        assertion.type = assertion.type.replace('promptfoo:redteam:', 'trusys:redteam:');
    }
    const baseType = inverse
        ? assertion.type.slice(4)
        : assertion.type;
    if (assertion.transform) {
        output = await (0, transform_1.transform)(assertion.transform, output, {
            vars: test.vars,
            prompt: { label: prompt },
        });
    }
    const context = {
        prompt,
        vars: test.vars || {},
        test,
        logProbs,
        provider,
        providerResponse,
        ...(assertion.config ? { config: structuredClone(assertion.config) } : {}),
    };
    // Render assertion values
    let renderedValue = assertion.value;
    let valueFromScript;
    if (typeof renderedValue === 'string') {
        if (renderedValue.startsWith('file://')) {
            const basePath = cliState_1.default.basePath || '';
            const fileRef = renderedValue.slice('file://'.length);
            let filePath = fileRef;
            let functionName;
            if (fileRef.includes(':')) {
                const [pathPart, funcPart] = fileRef.split(':');
                filePath = pathPart;
                functionName = funcPart;
            }
            filePath = path_1.default.resolve(basePath, filePath);
            if ((0, fileExtensions_1.isJavascriptFile)(filePath)) {
                valueFromScript = await (0, utils_1.loadFromJavaScriptFile)(filePath, functionName, [output, context]);
                logger_1.default.debug(`Javascript script ${filePath} output: ${valueFromScript}`);
            }
            else if (filePath.endsWith('.py')) {
                try {
                    const pythonScriptOutput = await (0, pythonUtils_1.runPython)(filePath, functionName || 'get_assert', [
                        output,
                        context,
                    ]);
                    valueFromScript = pythonScriptOutput;
                    logger_1.default.debug(`Python script ${filePath} output: ${valueFromScript}`);
                }
                catch (error) {
                    return {
                        pass: false,
                        score: 0,
                        reason: error.message,
                        assertion,
                    };
                }
            }
            else {
                renderedValue = (0, utils_1.processFileReference)(renderedValue);
            }
        }
        else if ((0, packageParser_1.isPackagePath)(renderedValue)) {
            const basePath = cliState_1.default.basePath || '';
            const requiredModule = await (0, packageParser_1.loadFromPackage)(renderedValue, basePath);
            if (typeof requiredModule !== 'function') {
                throw new Error(`Assertion malformed: ${renderedValue} must be a function. Received: ${typeof requiredModule}`);
            }
            valueFromScript = await Promise.resolve(requiredModule(output, context));
        }
        else {
            // It's a normal string value
            renderedValue = nunjucks.renderString(renderedValue, test.vars || {});
        }
    }
    else if (renderedValue && Array.isArray(renderedValue)) {
        // Process each element in the array
        renderedValue = renderedValue.map((v) => {
            if (typeof v === 'string') {
                if (v.startsWith('file://')) {
                    return (0, utils_1.processFileReference)(v);
                }
                return nunjucks.renderString(v, test.vars || {});
            }
            return v;
        });
    }
    const assertionParams = {
        assertion,
        baseType,
        context,
        cost,
        inverse,
        latencyMs,
        logProbs,
        output,
        outputString: (0, utils_1.coerceString)(output),
        prompt,
        provider,
        providerResponse,
        renderedValue,
        test: (0, utils_1.getFinalTest)(test, assertion),
        valueFromScript,
        workspaceId,
    };
    // handle audio response
    if (providerResponse.audio) {
        if (providerResponse.metadata?.audio_result === true) {
            assertionParams.evaluateResult = true;
            assertionParams.output = assertion.output ?? '';
        }
        else {
            assertionParams.outputAudio = providerResponse.audio;
            assertionParams.output = providerResponse.output ?? '';
        }
    }
    // handle video response
    if (providerResponse.video) {
        assertionParams.outputVideo = providerResponse.video;
        assertionParams.output = providerResponse.output ?? '';
    }
    const assertionHandlers = {
        'answer-relevance': answerRelevance_1.handleAnswerRelevance,
        bleu: bleu_1.handleBleuScore,
        classifier: classifier_1.handleClassifier,
        contains: contains_1.handleContains,
        'code-execution': codeExecuation_1.codeExecutionAssertion,
        'conversation-completeness': conversationCompletenessAssertion_1.handleConversationCompleteness,
        'knowledge-retention': knowledgeRetention_1.handleKnowledgeRetention,
        'conversational-g-eval': conversationalGEval_1.handleConversationalGEval,
        'conversation-relevancy': conversationRelevancyAssertion_1.handleConversationRelevancy,
        'role-adherence': handleRoleAdherence_1.handleRoleAdherence,
        'image-helpfulness': multimodalCommon_1.handleMultiModalCommon,
        'text-to-image': multimodalCommon_1.handleMultiModalCommon,
        'image-editing': imageEditing_1.handleImageEditing,
        'image-coherence': multimodalCommon_1.handleMultiModalCommon,
        'image-reference': multimodalCommon_1.handleMultiModalCommon,
        'multimodal-answer-relevancy': multimodalCommon_1.handleMultiModalCommon,
        'multimodal-faithfulness': handleMultimodalFaithfulness_1.handleMultimodalFaithfulness,
        'multimodal-g-eval': gEval_1.handleMultimodalGEval,
        'multimodal-contextual-precision': handleMultimodalContextualPrecision_1.handleMultimodalContextualPrecision,
        'multimodal-contextual-recall': handleMultimodalContextualRecall_1.handleMultimodalContextualRecall,
        'multimodal-contextual-relevancy': handleMultimodalContextualRelevancy_1.handleMultimodalContextualRelevancy,
        'code-correctness': handleCodeLanguage_1.handleCodeCorrectness,
        'contains-code-language': handleCodeLanguage_1.handleContainsCodeLanguage,
        'code-language': handleCodeLanguage_1.handleCodeLanguage,
        'code-readability': handleCodeLanguage_1.handleCodeReadability,
        hallucination: hallucination_1.handleHallucination,
        'contains-all': contains_1.handleContainsAll,
        'contains-any': contains_1.handleContainsAny,
        'contains-html': xml_1.handleIsXml,
        'contains-json': json_1.handleContainsJson,
        'contains-sql': sql_1.handleContainsSql,
        'contains-xml': xml_1.handleIsXml,
        'context-faithfulness': contextFaithfulness_1.handleContextFaithfulness,
        'context-recall': contextRecall_1.handleContextRecall,
        'context-relevance': contextRelevance_1.handleContextRelevance,
        'conversation-relevance': conversationRelevancyAssertion_1.handleConversationRelevancy,
        cost: cost_1.handleCost,
        equals: equals_1.handleEquals,
        factuality: factuality_1.handleFactuality,
        gleu: gleu_1.handleGleuScore,
        guardrails: guardrails_1.handleGuardrails,
        'g-eval': geval_1.handleGEval,
        icontains: contains_1.handleIContains,
        'icontains-all': contains_1.handleIContainsAll,
        'icontains-any': contains_1.handleIContainsAny,
        'is-html': xml_1.handleIsXml,
        'is-json': json_1.handleIsJson,
        'not-is-json': not_json_1.handleNotIsJson,
        'is-refusal': refusal_1.handleIsRefusal,
        'is-sql': sql_1.handleIsSql,
        'is-valid-function-call': functionToolCall_1.handleIsValidFunctionCall,
        'is-valid-openai-function-call': functionToolCall_1.handleIsValidFunctionCall,
        'is-valid-openai-tools-call': openai_1.handleIsValidOpenAiToolsCall,
        'is-xml': xml_1.handleIsXml,
        'finish-reason': ({ assertion, providerResponse, inverse }) => {
            const finishReason = providerResponse.finishReason || '';
            const expectedReason = typeof assertion.value === 'string' ? assertion.value : '';
            const pass = (finishReason === expectedReason) !== inverse;
            return {
                pass,
                score: pass ? 1 : 0,
                reason: pass
                    ? 'Assertion passed'
                    : `Expected finish reason to ${inverse ? 'not ' : ''}be "${expectedReason}", but got "${finishReason}"`,
                assertion,
            };
        },
        javascript: javascript_1.handleJavascript,
        latency: latency_1.handleLatency,
        levenshtein: levenshtein_1.handleLevenshtein,
        'llm-rubric': llmRubric_1.handleLlmRubric,
        meteor: async (params) => {
            try {
                const { handleMeteorAssertion } = await import('./meteor.js');
                return handleMeteorAssertion(params);
            }
            catch (error) {
                if (error instanceof Error && error.message.includes('Cannot find module')) {
                    return {
                        pass: false,
                        score: 0,
                        reason: 'METEOR assertion requires the natural package. Please install it using: npm install natural',
                        assertion: params.assertion,
                    };
                }
                throw error;
            }
        },
        'model-graded-closedqa': modelGradedClosedQa_1.handleModelGradedClosedQa,
        'model-graded-factuality': factuality_1.handleFactuality,
        moderation: moderation_1.handleModeration,
        perplexity: perplexity_1.handlePerplexity,
        'perplexity-score': perplexity_1.handlePerplexityScore,
        python: python_1.handlePython,
        regex: regex_1.handleRegex,
        'rouge-n': rouge_1.handleRougeScore,
        'not-rouge-n': not_rouge_1.handleNotRougeScore,
        'ruby': similar_1.handleSimilar, // Ruby uses similar handler (similarity-based)
        similar: similar_1.handleSimilar,
        'similar:cosine': similar_1.handleSimilar,
        'similar:dot': similar_1.handleSimilar,
        'similar:euclidean': similar_1.handleSimilar,
        'starts-with': startsWith_1.handleStartsWith,
        webhook: webhook_1.handleWebhook,
        pi: pi_1.handlePiScorer,
        'audio-sentiment': sentiment_1.handleSentiment,
        'audio-call-category': callCategory_1.handleCallCategory,
        'audio-interruption-handling': interruption_1.handleInterruption,
        'audio-language': language_1.handleLanguage,
        'audio-model-graded': combinedAudio_1.handleCombinedAudio,
        'audio-intent-identification': falseIntentIdentification_1.handleFalseIntentIdentification,
        'audio-consent-check': general_1.handleAudioGeneral,
        'audio-drop-out-rates': general_1.handleAudioGeneral,
        'audio-agent-response-time': responseTime_1.handleAgentResponseTime,
        'audio-talk-to-listen-ratio': talkToListenRatio_1.handleTalkToListenRatio,
        'audio-conversation-coherence': general_1.handleAudioGeneral,
        'audio-disclaimer-check': general_1.handleAudioGeneral,
        'audio-pii-check': general_1.handleAudioGeneral,
        'audio-language-switch': languageSwitch_1.handleLanguageSwitch,
        'audio-professional-tone': general_1.handleAudioGeneral,
        'audio-clarity': clarity_1.handleAudioClarity,
        'audio-background-noise': backgroundNoise_1.handleAudioBackgroundNoise,
        'audio-escalation': escalation_1.handleAudioEscalation,
        'audio-stt-accuracy': sttAcccuracy_1.handleSttAccuracy,
        'audio-stt-correctness': sttCorrectness_1.handleSttCorrectness,
        'audio-stt-missing': sttMissing_1.handleSttMissing,
        'audio-call-disconnected': callDisconnection_1.handleCallDisconnection,
        'audio-no-agent-response': noAgentResponse_1.handleNoAgentResponse,
        'audio-incomplete-agent-response': incompleteAgentResponse_1.handleIncompleteAgentResponse,
        'audio-repetition': repetition_1.handleRepetition,
        'audio-greeting': greeting_1.handleGreeting,
        'audio-number-correctness': sttNumberCorrectness_1.handleSttNumberCorrectness,
        'audio-call-outcome': callOutcome_1.handleCallOutcome,
        'tool-correctness': toolCorrectness_1.handleToolCorrectness,
        'task-completion': taskCompleteness_1.handleTaskCompleteness,
        'argument-correctness': argumentCorrectness_1.handleArgumentCorrectness,
        'plan-quality': planCorrectness_1.handlePlanCorrectness,
        'plan-adherence': planAdherence_1.handlePlanAdherence,
        'goal-fulfillment': goalFulfillment_1.handleGoalFulfillment,
        'execution-efficiency': executionEfficiency_1.handleExecutionEfficiency,
        'reasoning-coherence': reasoningCoherence_1.handleReasoningCoherence,
        'agent-custom-eval': agentCustomEval_1.handleAgentCustomEval,
        'tool-selection-quality': toolSelectionQuality_1.handleToolSelectionQuality,
        'tool-call-f1': toolCorrectness_1.handleToolCorrectness, // Use tool correctness handler
        'trace-error-spans': ({ assertion }) => ({
            pass: false,
            score: 0,
            reason: 'trace-error-spans assertion type is not yet implemented',
            assertion,
        }),
        'trace-span-count': ({ assertion }) => ({
            pass: false,
            score: 0,
            reason: 'trace-span-count assertion type is not yet implemented',
            assertion,
        }),
        'trace-span-duration': ({ assertion }) => ({
            pass: false,
            score: 0,
            reason: 'trace-span-duration assertion type is not yet implemented',
            assertion,
        }),
        'search-rubric': llmRubric_1.handleLlmRubric, // Use LLM rubric handler for search rubric
        'video-visual-quality': visualQuality_1.handleVisualQuality,
        'text-to-video': textToVideo_1.handleTextToVideo,
        'video-factual-consistency': factualConsistency_1.handleFactualConsistency,
        'video-temporal-consistency': temporalConsistency_1.handleTemporalConsistency,
        'video-g-eval': gEval_2.handleVideoGEval,
    };
    const handler = assertionHandlers[baseType];
    if (handler) {
        try {
            const result = await handler(assertionParams);
            // Check if the result itself contains an API error in the reason field
            if (isApiCallError(result.reason)) {
                return {
                    ...result,
                    metadata: {
                        ...result.metadata,
                        isApiError: true,
                    },
                };
            }
            // If weight is 0, treat this as a metric-only assertion that can't fail
            if (assertion.weight === 0) {
                return {
                    ...result,
                    pass: true, // Force pass for weight=0 assertions
                };
            }
            return result;
        }
        catch (error) {
            console.error('Error while running assertion: ', assertion.type, error);
            // Check if this is an API call error
            const errorMessage = error instanceof Error ? error.message : String(error);
            if (isApiCallError(errorMessage)) {
                return {
                    pass: false,
                    score: 0,
                    reason: errorMessage,
                    assertion,
                    // Mark this as an API error so it can be handled differently
                    metadata: { isApiError: true },
                };
            }
            else {
                return {
                    pass: false,
                    score: 0,
                    reason: errorMessage,
                    assertion,
                    // Mark this as an API error so it can be handled differently
                    metadata: { isAssertionError: true },
                };
            }
        }
    }
    if (baseType.startsWith('trusys:redteam:')) {
        try {
            return await (0, redteam_1.handleRedteam)(assertionParams);
        }
        catch (error) {
            console.error('Error while running redteam assertion: ', assertion.type, error);
            const errorMessage = error instanceof Error ? error.message : String(error);
            if (isApiCallError(errorMessage)) {
                return {
                    pass: false,
                    score: 0,
                    reason: errorMessage,
                    assertion,
                    // Mark this as an API error so it can be handled differently
                    metadata: { isApiError: true },
                };
            }
            else {
                return {
                    pass: false,
                    score: 0,
                    reason: errorMessage,
                    assertion,
                    // Mark this as an API error so it can be handled differently
                    metadata: { isAssertionError: true },
                };
            }
        }
    }
    throw new Error(`Unknown assertion type: ${assertion.type}`);
}
function addBatchedAudioAssertionFailures(mainAssertResult, batchedAssertions, reason, tokensUsed) {
    batchedAssertions.forEach((audioAssertion, i) => {
        mainAssertResult.addResult({
            index: audioAssertion.index,
            result: {
                pass: false,
                score: 0,
                reason,
                assertion: audioAssertion.assertion,
                ...(i === 0 && tokensUsed ? { tokensUsed } : {}),
            },
            metric: audioAssertion.assertion.metric,
            weight: audioAssertion.assertion.weight,
        });
    });
}
async function runAssertions({ assertScoringFunction, latencyMs, prompt, provider, providerResponse, test, workspaceId, }) {
    if (!test.assert || test.assert.length < 1) {
        return assertionsResult_1.AssertionsResult.noAssertsResult();
    }
    const mainAssertResult = new assertionsResult_1.AssertionsResult({
        threshold: test.threshold,
    });
    const subAssertResults = [];
    const asserts = test.assert
        .map((assertion, i) => {
        if (assertion.type === 'assert-set') {
            const subAssertResult = new assertionsResult_1.AssertionsResult({
                threshold: assertion.threshold,
                parentAssertionSet: {
                    assertionSet: assertion,
                    index: i,
                },
            });
            subAssertResults.push(subAssertResult);
            return assertion.assert.map((subAssert, j) => {
                return {
                    assertion: subAssert,
                    assertResult: subAssertResult,
                    index: j,
                };
            });
        }
        return { assertion, assertResult: mainAssertResult, index: i };
    })
        .flat();
    const combinedAssertions = {};
    await async_1.default.forEachOfLimit(asserts, ASSERTIONS_MAX_CONCURRENCY, async ({ assertion, assertResult, index }) => {
        if (assertion.type.startsWith('select-')) {
            // Select-type assertions are handled separately because they depend on multiple outputs.
            return;
        }
        const result = await runAssertion({
            prompt,
            provider,
            providerResponse,
            assertion,
            test,
            latencyMs,
            assertIndex: index,
            workspaceId,
        });
        if (types_1.AudioAssertionTypesSchema.options.includes(assertion.type) &&
            assertion.type !== 'audio-model-graded' &&
            assertion.type !== 'audio-stt-accuracy') {
            if (!combinedAssertions.audio) {
                combinedAssertions.audio = [];
            }
            combinedAssertions.audio.push({
                type: assertion.type,
                prompt: result.metadata?.prompt,
                metric: result.metadata?.metric,
                index,
                assertion,
            });
        }
        else {
            assertResult.addResult({
                index,
                result,
                metric: assertion.metric,
                weight: assertion.weight,
            });
        }
    });
    // Handle audio assertions
    const batchedAudioAssertions = combinedAssertions.audio ?? [];
    if (batchedAudioAssertions.length > 0) {
        const audioResult = await runAssertion({
            prompt: JSON.stringify(batchedAudioAssertions),
            provider,
            providerResponse,
            assertion: { type: 'audio-model-graded' },
            test,
            latencyMs,
            workspaceId,
        });
        logger_1.default.debug(`audioResult, ${JSON.stringify(audioResult)}`);
        const rawCombinedResult = audioResult.metadata?.result;
        const hasCombinedJson = typeof rawCombinedResult === 'string' && rawCombinedResult.trim().length > 0;
        if (hasCombinedJson) {
            let audioEvals;
            try {
                audioEvals = JSON.parse(rawCombinedResult);
            }
            catch (err) {
                const reason = `Combined audio grading returned invalid JSON: ${err instanceof Error ? err.message : String(err)}`;
                addBatchedAudioAssertionFailures(mainAssertResult, batchedAudioAssertions, reason, audioResult.tokensUsed);
            }
            if (audioEvals) {
                providerResponse.metadata = providerResponse.metadata ?? {};
                providerResponse.metadata.audio_result = true;
                if (audioEvals.transcript !== undefined) {
                    providerResponse.metadata.audio_transcript = audioEvals.transcript;
                }
                for (const audioAssertion of batchedAudioAssertions) {
                    const assertionType = audioAssertion.type;
                    if (!assertionType) {
                        addBatchedAudioAssertionFailures(mainAssertResult, [audioAssertion], 'Audio assertion missing type');
                        continue;
                    }
                    const key = assertionType.replace('audio-', '');
                    const metricOutput = audioEvals[key];
                    if (metricOutput === undefined) {
                        mainAssertResult.addResult({
                            index: audioAssertion.index,
                            result: {
                                pass: false,
                                score: 0,
                                reason: `Combined audio grading did not return metric "${key}"`,
                                assertion: audioAssertion.assertion,
                            },
                            metric: audioAssertion.assertion.metric,
                            weight: audioAssertion.assertion.weight,
                        });
                        continue;
                    }
                    audioAssertion.assertion.output = metricOutput;
                    const audioevalResult = await runAssertion({
                        prompt: '',
                        provider,
                        providerResponse,
                        assertion: audioAssertion.assertion,
                        test,
                        latencyMs,
                        workspaceId,
                    });
                    mainAssertResult.addResult({
                        index: audioAssertion.index,
                        result: audioevalResult,
                        metric: audioAssertion.assertion.metric,
                        weight: audioAssertion.assertion.weight,
                    });
                }
            }
        }
        else {
            const reason = audioResult.reason?.trim() ||
                'Combined audio grading did not return an analysis result (check STT, audio eval deployment, and API keys)';
            addBatchedAudioAssertionFailures(mainAssertResult, batchedAudioAssertions, reason, audioResult.tokensUsed);
        }
    }
    await async_1.default.forEach(subAssertResults, async (subAssertResult) => {
        const result = await subAssertResult.testResult();
        const { index, assertionSet: { metric, weight }, } = subAssertResult.parentAssertionSet;
        mainAssertResult.addResult({
            index,
            result,
            metric,
            weight,
        });
    });
    return mainAssertResult.testResult(assertScoringFunction);
}
async function runCompareAssertion(test, assertion, outputs, workspaceId) {
    (0, invariant_1.default)(typeof assertion.value === 'string', 'select-best must have a string value');
    test = (0, utils_1.getFinalTest)(test, assertion);
    const comparisonResults = await (0, matchers_1.matchesSelectBest)(assertion.value, outputs, test.options, test.vars, workspaceId);
    return comparisonResults.map((result) => ({
        ...result,
        assertion,
    }));
}
async function readAssertions(filePath) {
    try {
        const assertions = js_yaml_1.default.load(fs_1.default.readFileSync(filePath, 'utf-8'));
        if (!Array.isArray(assertions) || assertions[0]?.type === undefined) {
            throw new Error('Assertions file must be an array of assertion objects');
        }
        return assertions;
    }
    catch (err) {
        throw new Error(`Failed to read assertions from ${filePath}:\n${err}`);
    }
}
// These exports are used by the node.js package (index.ts)
exports.default = {
    runAssertion,
    runAssertions,
    matchesSimilarity: matchers_1.matchesSimilarity,
    matchesClassification: matchers_1.matchesClassification,
    matchesLlmRubric: matchers_1.matchesLlmRubric,
    matchesFactuality: matchers_1.matchesFactuality,
    matchesClosedQa: matchers_1.matchesClosedQa,
    matchesAnswerRelevance: matchers_1.matchesAnswerRelevance,
    matchesContextRecall: matchers_1.matchesContextRecall,
    matchesContextRelevance: matchers_1.matchesContextRelevance,
    matchesContextFaithfulness: matchers_1.matchesContextFaithfulness,
    matchesComparisonBoolean: matchers_1.matchesSelectBest,
    matchesModeration: matchers_1.matchesModeration,
};
//# sourceMappingURL=index.js.map