"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.handleCodeReadability = exports.CODE_LANGUAGE_PROMPT = void 0;
exports.getGradingProvider = getGradingProvider;
exports.getAndCheckProvider = getAndCheckProvider;
exports.matchesSimilarity = matchesSimilarity;
exports.matchesClassification = matchesClassification;
exports.renderLlmRubricPrompt = renderLlmRubricPrompt;
exports.matchesLlmRubric = matchesLlmRubric;
exports.matchesPiScore = matchesPiScore;
exports.matchesFactuality = matchesFactuality;
exports.matchesClosedQa = matchesClosedQa;
exports.matchesGEval = matchesGEval;
exports.matchesMultimodalGEval = matchesMultimodalGEval;
exports.matchesAnswerRelevance = matchesAnswerRelevance;
exports.matchesContextRecall = matchesContextRecall;
exports.matchesContextRelevance = matchesContextRelevance;
exports.matchesContextFaithfulness = matchesContextFaithfulness;
exports.matchesPlanCorrectness = matchesPlanCorrectness;
exports.matchesPlanAdherence = matchesPlanAdherence;
exports.matchesGoalFulfillment = matchesGoalFulfillment;
exports.matchesToolSelectionQuality = matchesToolSelectionQuality;
exports.matchesExecutionEfficiency = matchesExecutionEfficiency;
exports.matchesReasoningCoherence = matchesReasoningCoherence;
exports.matchesAgentCustomEval = matchesAgentCustomEval;
exports.matchesSelectBest = matchesSelectBest;
exports.matchesModeration = matchesModeration;
exports.matchesSentiment = matchesSentiment;
exports.matchesCodeCorrectness = matchesCodeCorrectness;
exports.detectCodeLanguage = detectCodeLanguage;
exports.matchesContainsCodeLanguage = matchesContainsCodeLanguage;
exports.matchesCodeReadability = matchesCodeReadability;
const dedent_1 = __importDefault(require("dedent"));
const path_1 = __importDefault(require("path"));
const utils_1 = require("./assertions/utils");
const cliState_1 = __importDefault(require("./cliState"));
const envars_1 = require("./envars");
const logger_1 = __importDefault(require("./logger"));
const prompts_1 = require("./prompts");
const multiModal_1 = require("./prompts/multiModal");
const providers_1 = require("./providers");
const defaults_1 = require("./providers/defaults");
const constants_1 = require("./redteam/constants");
const remoteGeneration_1 = require("./redteam/remoteGeneration");
const remoteGrading_1 = require("./remoteGrading");
const remoteScoring_1 = require("./remoteScoring");
const file_1 = require("./util/file");
const fileExtensions_1 = require("./util/fileExtensions");
const invariant_1 = __importDefault(require("./util/invariant"));
const json_1 = require("./util/json");
const templates_1 = require("./util/templates");
const lodash_1 = __importDefault(require("lodash"));
class LlmRubricProviderError extends Error {
    constructor(message) {
        super(message);
        this.name = 'LlmRubricProviderError';
    }
}
const nunjucks = (0, templates_1.getNunjucksEngine)(undefined, false, true);
function cosineSimilarity(vecA, vecB) {
    if (vecA.length !== vecB.length) {
        throw new Error('Vectors must be of equal length');
    }
    const dotProduct = vecA.reduce((acc, val, idx) => acc + val * vecB[idx], 0);
    const vecAMagnitude = Math.sqrt(vecA.reduce((acc, val) => acc + val * val, 0));
    const vecBMagnitude = Math.sqrt(vecB.reduce((acc, val) => acc + val * val, 0));
    return dotProduct / (vecAMagnitude * vecBMagnitude);
}
function fromVars(vars) {
    if (!vars) {
        return {};
    }
    const ret = {};
    for (const [key, value] of Object.entries(vars)) {
        if (typeof value === 'object') {
            ret[key] = JSON.stringify(value);
        }
        else {
            ret[key] = JSON.stringify(value).slice(1, -1);
        }
    }
    return ret;
}
async function loadFromProviderOptions(provider) {
    (0, invariant_1.default)(typeof provider === 'object', `Provider must be an object, but received a ${typeof provider}: ${provider}`);
    (0, invariant_1.default)(!Array.isArray(provider), `Provider must be an object, but received an array: ${JSON.stringify(provider)}`);
    (0, invariant_1.default)(provider.id, 'Provider supplied to assertion must have an id');
    // TODO(ian): set basepath if invoked from filesystem config
    return (0, providers_1.loadApiProvider)(provider.id, { options: provider });
}
async function getGradingProvider(type, provider, defaultProvider) {
    let finalProvider;
    if (typeof provider === 'string') {
        // Defined as a string
        finalProvider = await (0, providers_1.loadApiProvider)(provider);
    }
    else if (typeof provider === 'object' && typeof provider.id === 'function') {
        // Defined as an ApiProvider interface
        finalProvider = provider;
    }
    else if (typeof provider === 'object') {
        const typeValue = provider[type];
        if (typeValue) {
            // Defined as embedding, classification, or text record
            finalProvider = await getGradingProvider(type, typeValue, defaultProvider);
        }
        else if (provider.id) {
            // Defined as ProviderOptions
            finalProvider = await loadFromProviderOptions(provider);
        }
        else if (Array.isArray(provider)) {
            throw new Error(`Provider must be an object or string, but received an array.\n\nCheck that the provider ${JSON.stringify(provider[0], null, 2)} is not nested in an array.`);
        }
        else {
            throw new Error(`Invalid provider definition for output type '${type}': ${JSON.stringify(provider, null, 2)}`);
        }
    }
    else {
        finalProvider = defaultProvider;
    }
    return finalProvider;
}
async function getAndCheckProvider(type, provider, defaultProvider, checkName) {
    const matchedProvider = await getGradingProvider(type, provider, defaultProvider);
    if (!matchedProvider) {
        if (defaultProvider) {
            logger_1.default.warn(`No provider of type ${type} found for '${checkName}', falling back to default`);
            return defaultProvider;
        }
        else {
            throw new Error(`No provider of type ${type} found for '${checkName}'`);
        }
    }
    let isValidProviderType = true;
    if (type === 'embedding') {
        isValidProviderType =
            'callEmbeddingApi' in matchedProvider || 'callSimilarityApi' in matchedProvider;
    }
    else if (type === 'classification') {
        isValidProviderType = 'callClassificationApi' in matchedProvider;
    }
    else if (type === 'moderation') {
        isValidProviderType = 'callModerationApi' in matchedProvider;
    }
    if (!isValidProviderType) {
        if (defaultProvider) {
            logger_1.default.warn(`Provider ${matchedProvider.id()} is not a valid ${type} provider for '${checkName}', falling back to default`);
            return defaultProvider;
        }
        else {
            throw new Error(`Provider ${matchedProvider.id()} is not a valid ${type} provider for '${checkName}'`);
        }
    }
    return matchedProvider;
}
function fail(reason, tokensUsed) {
    return {
        pass: false,
        reason,
        score: 0,
        tokensUsed: {
            total: tokensUsed?.total || 0,
            prompt: tokensUsed?.prompt || 0,
            completion: tokensUsed?.completion || 0,
            cached: tokensUsed?.cached || 0,
            completionDetails: tokensUsed?.completionDetails,
        },
    };
}
async function matchesSimilarity(expected, output, threshold, inverse = false, grading, workspaceId) {
    if (cliState_1.default.config?.redteam && (0, remoteGeneration_1.shouldGenerateRemote)()) {
        try {
            return (0, remoteGrading_1.doRemoteGrading)({
                task: 'similar',
                expected,
                output,
                threshold,
                inverse,
            });
        }
        catch (error) {
            return fail(`Could not perform remote grading: ${error}`);
        }
    }
    const finalProvider = (await getAndCheckProvider('embedding', grading?.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).embeddingProvider, 'similarity check'));
    let similarity;
    const tokensUsed = {
        total: 0,
        prompt: 0,
        completion: 0,
        cached: 0,
        model: 'text-embedding-3-small',
        completionDetails: {
            reasoning: 0,
            acceptedPrediction: 0,
            rejectedPrediction: 0,
        },
    };
    if ('callSimilarityApi' in finalProvider) {
        const similarityResp = await finalProvider.callSimilarityApi(expected, output);
        tokensUsed.total = similarityResp.tokenUsage?.total || 0;
        tokensUsed.prompt = similarityResp.tokenUsage?.prompt || 0;
        tokensUsed.completion = similarityResp.tokenUsage?.completion || 0;
        tokensUsed.cached = similarityResp.tokenUsage?.cached || 0;
        tokensUsed.completionDetails = similarityResp.tokenUsage?.completionDetails;
        if (similarityResp.error) {
            return fail(similarityResp.error, tokensUsed);
        }
        if (similarityResp.similarity == null) {
            return fail('Unknown error fetching similarity', tokensUsed);
        }
        similarity = similarityResp.similarity;
    }
    else if ('callEmbeddingApi' in finalProvider) {
        const expectedEmbedding = await finalProvider.callEmbeddingApi(expected);
        const outputEmbedding = await finalProvider.callEmbeddingApi(output);
        tokensUsed.total =
            (expectedEmbedding.tokenUsage?.total || 0) + (outputEmbedding.tokenUsage?.total || 0);
        tokensUsed.prompt =
            (expectedEmbedding.tokenUsage?.prompt || 0) + (outputEmbedding.tokenUsage?.prompt || 0);
        tokensUsed.completion =
            (expectedEmbedding.tokenUsage?.completion || 0) +
                (outputEmbedding.tokenUsage?.completion || 0);
        tokensUsed.cached =
            (expectedEmbedding.tokenUsage?.cached || 0) + (outputEmbedding.tokenUsage?.cached || 0);
        tokensUsed.model = 'text-embedding-3-small';
        tokensUsed.completionDetails = {
            reasoning: (expectedEmbedding.tokenUsage?.completionDetails?.reasoning || 0) +
                (outputEmbedding.tokenUsage?.completionDetails?.reasoning || 0),
            acceptedPrediction: (expectedEmbedding.tokenUsage?.completionDetails?.acceptedPrediction || 0) +
                (outputEmbedding.tokenUsage?.completionDetails?.acceptedPrediction || 0),
            rejectedPrediction: (expectedEmbedding.tokenUsage?.completionDetails?.rejectedPrediction || 0) +
                (outputEmbedding.tokenUsage?.completionDetails?.rejectedPrediction || 0),
        };
        if (expectedEmbedding.error || outputEmbedding.error) {
            return fail(expectedEmbedding.error || outputEmbedding.error || 'Unknown error fetching embeddings', tokensUsed);
        }
        if (!expectedEmbedding.embedding || !outputEmbedding.embedding) {
            return fail('Embedding not found', tokensUsed);
        }
        similarity = cosineSimilarity(expectedEmbedding.embedding, outputEmbedding.embedding);
    }
    else {
        throw new Error('Provider must implement callSimilarityApi or callEmbeddingApi');
    }
    const pass = inverse
        ? similarity <= threshold + Number.EPSILON
        : similarity >= threshold - Number.EPSILON;
    const greaterThanReason = `Similarity ${similarity.toFixed(2)} is greater than threshold ${threshold}`;
    const lessThanReason = `Similarity ${similarity.toFixed(2)} is less than threshold ${threshold}`;
    if (pass) {
        return {
            pass: true,
            score: inverse ? 1 - similarity : similarity,
            reason: inverse ? lessThanReason : greaterThanReason,
            tokensUsed,
        };
    }
    return {
        pass: false,
        score: inverse ? 1 - similarity : similarity,
        reason: inverse ? greaterThanReason : lessThanReason,
        tokensUsed,
    };
}
/**
 *
 * @param expected Expected classification. If undefined, matches any classification.
 * @param output Text to classify.
 * @param threshold Value between 0 and 1. If the expected classification is undefined, the threshold is the minimum score for any classification. If the expected classification is defined, the threshold is the minimum score for that classification.
 * @param grading
 * @returns Pass if the output matches the classification with a score greater than or equal to the threshold.
 */
async function matchesClassification(expected, output, threshold, grading) {
    const finalProvider = (await getAndCheckProvider('classification', grading?.provider, null, 'classification check'));
    const resp = await finalProvider.callClassificationApi(output);
    if (!resp.classification) {
        return fail(resp.error || 'Unknown error fetching classification');
    }
    let score;
    if (expected === undefined) {
        score = Math.max(...Object.values(resp.classification));
    }
    else {
        score = resp.classification[expected] || 0;
    }
    if (score >= threshold - Number.EPSILON) {
        const reason = expected === undefined
            ? `Maximum classification score ${score.toFixed(2)} >= ${threshold}`
            : `Classification ${expected} has score ${score.toFixed(2)} >= ${threshold}`;
        return {
            pass: true,
            score,
            reason,
        };
    }
    return {
        pass: false,
        score,
        reason: `Classification ${expected} has score ${score.toFixed(2)} < ${threshold}`,
    };
}
async function loadRubricPrompt(rubricPrompt, defaultPrompt) {
    if (!rubricPrompt ||
        (typeof rubricPrompt === 'object' && Object.keys(rubricPrompt ?? {}).length === 0)) {
        return defaultPrompt;
    }
    if (typeof rubricPrompt === 'string' &&
        rubricPrompt.startsWith('file://') &&
        (0, fileExtensions_1.isJavascriptFile)(rubricPrompt)) {
        const basePath = cliState_1.default.basePath || '';
        let filePath = rubricPrompt.slice('file://'.length);
        const [pathPart, functionName] = filePath.split(':');
        filePath = path_1.default.resolve(basePath, pathPart);
        rubricPrompt = await (0, utils_1.loadFromJavaScriptFile)(filePath, functionName, []);
    }
    else {
        // Load from external file if needed
        rubricPrompt = (0, file_1.maybeLoadFromExternalFile)(rubricPrompt);
    }
    if (typeof rubricPrompt === 'object') {
        rubricPrompt = JSON.stringify(rubricPrompt);
    }
    (0, invariant_1.default)(typeof rubricPrompt === 'string', 'rubricPrompt must be a string');
    return rubricPrompt;
}
function tryParse(content) {
    try {
        return JSON.parse(content);
    }
    catch { }
    return content;
}
function splitIntoSentences(text) {
    return text.split('\n').filter((sentence) => sentence.trim() !== '');
}
function processContextForTemplating(context, enableObjectAccess) {
    if (enableObjectAccess) {
        return context;
    }
    return Object.fromEntries(Object.entries(context).map(([key, value]) => {
        if (value && typeof value === 'object') {
            if (Array.isArray(value)) {
                return [
                    key,
                    value.map((item) => (item && typeof item === 'object' ? JSON.stringify(item) : item)),
                ];
            }
            return [key, JSON.stringify(value)];
        }
        return [key, value];
    }));
}
async function renderLlmRubricPrompt(rubricPrompt, context) {
    const enableObjectAccess = (0, envars_1.getEnvBool)('TRUSYS_DISABLE_OBJECT_STRINGIFY', false);
    const processedContext = processContextForTemplating(context, enableObjectAccess);
    try {
        // Render every string scalar within the JSON
        // Does not render object keys (only values)
        const parsed = JSON.parse(rubricPrompt, (k, v) => typeof v === 'string' ? nunjucks.renderString(v, processedContext) : v);
        return JSON.stringify(parsed);
    }
    catch {
        // not valid JSON...
        // output a warning?
    }
    // Legacy rendering for non-JSON prompts
    return nunjucks.renderString(rubricPrompt, processedContext);
}
async function matchesLlmRubric(rubric, llmOutput, grading, vars, assertion, options, workspaceId) {
    if (!grading) {
        throw new Error('Cannot grade output without grading config. Specify --grader option or grading config.');
    }
    if (!grading.rubricPrompt && cliState_1.default.config?.redteam && (0, remoteGeneration_1.shouldGenerateRemote)()) {
        return {
            ...(await (0, remoteGrading_1.doRemoteGrading)({
                task: 'llm-rubric',
                rubric,
                output: llmOutput,
                vars: vars || {},
            })),
            assertion: assertion ?? undefined,
        };
    }
    const rubricPrompt = await loadRubricPrompt(grading?.rubricPrompt, prompts_1.DEFAULT_GRADING_PROMPT);
    const prompt = await renderLlmRubricPrompt(rubricPrompt, {
        output: tryParse(llmOutput),
        rubric,
        ...(vars || {}),
    });
    const defaultProviders = await (0, defaults_1.getDefaultProviders)(undefined, workspaceId);
    const defaultProvider = defaultProviders.llmRubricProvider || defaultProviders.gradingJsonProvider;
    const finalProvider = await getAndCheckProvider('text', grading.provider, defaultProvider, 'llm-rubric check');
    const resp = await finalProvider.callApi(prompt);
    if (resp.error || !resp.output) {
        if (options?.throwOnError) {
            throw new LlmRubricProviderError(resp.error || 'No output');
        }
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    let jsonObjects = [];
    if (typeof resp.output === 'string') {
        try {
            jsonObjects = (0, json_1.extractJsonObjects)(resp.output);
            if (jsonObjects.length === 0) {
                return fail('Could not extract JSON from llm-rubric response', resp.tokenUsage);
            }
        }
        catch (err) {
            return fail(`llm-rubric produced malformed response: ${err}\n\n${resp.output}`, resp.tokenUsage);
        }
    }
    else if (typeof resp.output === 'object') {
        jsonObjects = [resp.output];
    }
    else {
        return fail('llm-rubric produced malformed response - output must be string or object', resp.tokenUsage);
    }
    // expects properties pass, score, and reason
    const parsed = jsonObjects[0]; // as Partial<GradingResult>;
    let pass = parsed.pass ?? true;
    if (typeof pass !== 'boolean') {
        pass = /^(true|yes|pass|y)$/i.test(String(pass));
    }
    let score = parsed.score;
    if (typeof score !== 'number') {
        score = Number.isFinite(Number(score)) ? Number(score) : Number(pass);
    }
    const threshold = typeof assertion?.threshold === 'string' ? Number(assertion.threshold) : assertion?.threshold;
    if (typeof threshold === 'number' && Number.isFinite(threshold)) {
        pass = pass && score >= threshold;
    }
    const reason = parsed.reason || (pass ? 'Grading passed' : `Score ${score} below threshold ${threshold}`);
    const recommendation = parsed.recommendation;
    return {
        assertion: assertion ?? undefined,
        pass,
        score,
        reason,
        suggestions: [{
                action: 'note',
                type: 'recommendation',
                value: recommendation || '',
            }],
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
            cached: resp.tokenUsage?.cached || 0,
            model: resp.tokenUsage?.model || '',
            completionDetails: parsed.tokensUsed?.completionDetails || {
                reasoning: 0,
                acceptedPrediction: 0,
                rejectedPrediction: 0,
            },
        },
    };
}
async function matchesPiScore(renderedValue, llmInput, llmOutput, assertion) {
    return {
        ...(await (0, remoteScoring_1.doRemoteScoringWithPi)({
            llm_input: llmInput,
            llm_output: llmOutput,
            scoring_spec: [
                {
                    question: renderedValue,
                },
            ],
        }, assertion?.threshold)),
        assertion: assertion ?? undefined,
    };
}
async function matchesFactuality(input, expected, output, grading, vars, workspaceId) {
    if (!grading) {
        throw new Error('Cannot grade output without grading config. Specify --grader option or grading config.');
    }
    const rubricPrompt = await loadRubricPrompt(grading?.rubricPrompt, prompts_1.TRUSYS_FACTUALITY_PROMPT);
    const prompt = await renderLlmRubricPrompt(rubricPrompt, {
        input,
        ideal: expected,
        completion: tryParse(output),
        ...(vars || {}),
    });
    // Get the appropriate provider
    const finalProvider = await getAndCheckProvider('text', grading.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).gradingProvider, 'factuality check');
    const resp = await finalProvider.callApi(prompt);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'factuality produced malformed response');
    // Copied from standard factuality grading prompt
    const categoryDescriptions = {
        A: 'The submitted answer is a subset of the expert answer and is fully consistent with it.',
        B: 'The submitted answer is a superset of the expert answer and is fully consistent with it.',
        C: 'The submitted answer contains all the same details as the expert answer.',
        D: 'There is a disagreement between the submitted answer and the expert answer.',
        E: "The answers differ, but these differences don't matter from the perspective of factuality.",
    };
    // Try to parse as JSON first
    let jsonData = null;
    let jsonError = null;
    try {
        jsonData = (0, json_1.extractFirstJsonObject)(resp.output);
    }
    catch (err) {
        jsonError = err;
        logger_1.default.debug(`JSON parsing failed: ${jsonError.message}`);
    }
    // If JSON parsing succeeded and provided a valid category
    if (jsonData && jsonData.category && typeof jsonData.category === 'string') {
        const option = jsonData.category.trim().toUpperCase();
        if (!/^[A-E]$/.test(option)) {
            return fail(`Invalid category value: ${option}`, resp.tokenUsage);
        }
        const scoreLookup = {
            A: grading.factuality?.subset ?? 1,
            B: grading.factuality?.superset ?? 1,
            C: grading.factuality?.agree ?? 1,
            D: grading.factuality?.disagree ?? 0,
            E: grading.factuality?.differButFactual ?? 1,
        };
        // Determine if this option passes or fails
        const passing = Object.keys(scoreLookup).filter((key) => scoreLookup[key] > 0);
        const failing = Object.keys(scoreLookup).filter((key) => scoreLookup[key] === 0);
        const pass = passing.includes(option) && !failing.includes(option);
        // Use the model's reason if available, otherwise fall back to the category description
        const modelReason = jsonData.reason?.trim();
        const reason = modelReason || `Category ${option}: ${categoryDescriptions[option]}`;
        const score = scoreLookup[option] ?? (pass ? 1 : 0);
        return {
            pass,
            score,
            reason,
            tokensUsed: resp.tokenUsage || {
                total: 0,
                prompt: 0,
                completion: 0,
                cached: 0,
                duration: 0,
                model: '',
                completionDetails: {
                    reasoning: 0,
                    acceptedPrediction: 0,
                    rejectedPrediction: 0,
                },
            },
        };
    }
    // Fallback to old pattern matching format
    logger_1.default.info('Falling back to legacy pattern matching for factuality check');
    const responseText = resp.output;
    // The preferred output starts like "(A)...", but we also support leading whitespace, lowercase letters, and omitting the first parenthesis.
    const answerMatch = responseText.match(/\s*\(?([a-eA-E])\)/);
    if (!answerMatch) {
        return fail(`Factuality checker output did not match expected format: ${responseText}`, resp.tokenUsage);
    }
    const option = answerMatch[1].toUpperCase();
    let modelReason = responseText;
    const reasonMatch = responseText.match(/\)\s*(.*)/s);
    if (reasonMatch && reasonMatch[1]) {
        modelReason = reasonMatch[1].trim();
    }
    const scoreLookup = {
        A: grading.factuality?.subset ?? 1,
        B: grading.factuality?.superset ?? 1,
        C: grading.factuality?.agree ?? 1,
        D: grading.factuality?.disagree ?? 0,
        E: grading.factuality?.differButFactual ?? 1,
    };
    const passing = Object.keys(scoreLookup).filter((key) => scoreLookup[key] > 0);
    const failing = Object.keys(scoreLookup).filter((key) => scoreLookup[key] === 0);
    const pass = passing.includes(option) && !failing.includes(option);
    const score = scoreLookup[option] ?? (pass ? 1 : 0);
    return {
        pass,
        score,
        reason: modelReason,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
            cached: resp.tokenUsage?.cached || 0,
            duration: resp.tokenUsage?.duration || 0,
            model: resp.tokenUsage?.model || '',
            completionDetails: resp.tokenUsage?.completionDetails || {
                reasoning: 0,
                acceptedPrediction: 0,
                rejectedPrediction: 0,
            },
        },
    };
}
async function matchesClosedQa(input, expected, output, grading, vars, workspaceId) {
    if (!grading) {
        throw new Error('Cannot grade output without grading config. Specify --grader option or grading config.');
    }
    const rubricPrompt = await loadRubricPrompt(grading?.rubricPrompt, prompts_1.OPENAI_CLOSED_QA_PROMPT);
    const prompt = await renderLlmRubricPrompt(rubricPrompt, {
        input,
        criteria: expected,
        completion: tryParse(output),
        ...(vars || {}),
    });
    const finalProvider = await getAndCheckProvider('text', grading.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).gradingProvider, 'model-graded-closedqa check');
    const resp = await finalProvider.callApi(prompt);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'model-graded-closedqa produced malformed response');
    try {
        const pass = resp.output.trimEnd().endsWith('Y');
        let reason;
        if (pass) {
            reason = 'The submission meets the criterion';
        }
        else if (resp.output.trimEnd().endsWith('N')) {
            reason = `The submission does not meet the criterion:\n${resp.output}`;
        }
        else {
            reason = `Model grader produced a malformed response:\n${resp.output}`;
        }
        return {
            pass,
            score: pass ? 1 : 0,
            reason,
            tokensUsed: {
                total: resp.tokenUsage?.total || 0,
                prompt: resp.tokenUsage?.prompt || 0,
                completion: resp.tokenUsage?.completion || 0,
                cached: resp.tokenUsage?.cached || 0,
                model: resp.tokenUsage?.model || '',
                completionDetails: resp.tokenUsage?.completionDetails || {
                    reasoning: 0,
                    acceptedPrediction: 0,
                    rejectedPrediction: 0,
                },
            },
        };
    }
    catch (err) {
        return fail(`Error parsing output: ${err.message}`, resp.tokenUsage);
    }
}
async function matchesGEval(criteria, input, output, threshold, grading, workspaceId) {
    if (!input) {
        throw Error('No source text to estimate reply');
    }
    const maxScore = 10;
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).gradingProvider, 'reply geval check');
    // Step 1: Get evaluation steps using renderLlmRubricPrompt
    const stepsRubricPrompt = typeof grading?.rubricPrompt === 'object' && !Array.isArray(grading?.rubricPrompt)
        ? grading?.rubricPrompt?.['steps']
        : undefined;
    const stepsPrompt = await loadRubricPrompt(stepsRubricPrompt, prompts_1.GEVAL_PROMPT_STEPS);
    const promptSteps = await renderLlmRubricPrompt(stepsPrompt, { criteria });
    const respSteps = await textProvider.callApi(promptSteps);
    let steps;
    try {
        // NOTE: use regexp for reliable, because sometimes LLM wraps response to markdown format ```json...```
        steps = JSON.parse(respSteps.output.match(/\{"steps".+\}/g)[0]).steps;
        if (!steps.length) {
            return fail('LLM does not propose any evaluation step');
        }
    }
    catch {
        return fail(`LLM-proposed evaluation steps are not in JSON format: ${respSteps.output}`);
    }
    // Step 2: Use steps to evaluate using renderLlmRubricPrompt
    const evalRubricPrompt = typeof grading?.rubricPrompt === 'object' && !Array.isArray(grading?.rubricPrompt)
        ? grading?.rubricPrompt?.['evaluate']
        : undefined;
    const evalPrompt = await loadRubricPrompt(evalRubricPrompt, prompts_1.GEVAL_PROMPT_EVALUATE);
    const promptText = await renderLlmRubricPrompt(evalPrompt, {
        criteria,
        steps: steps.join('\n- '),
        maxScore: maxScore.toString(),
        input: tryParse(input),
        output: tryParse(output),
    });
    const resp = await textProvider.callApi(promptText);
    let result;
    try {
        result = JSON.parse(resp.output.match(/\{.+\}/g)[0]);
    }
    catch {
        return fail(`LLM-proposed evaluation result is not in JSON format: ${resp.output}`);
    }
    return {
        pass: result.score / maxScore >= threshold,
        score: result.score / maxScore,
        reason: result.reason,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
            cached: resp.tokenUsage?.cached || 0,
            model: resp.tokenUsage?.model || '',
            completionDetails: resp.tokenUsage?.completionDetails || {
                reasoning: 0,
                acceptedPrediction: 0,
                rejectedPrediction: 0,
            },
        },
    };
}
async function matchesMultimodalGEval(criteria, input, output, threshold, evalType, grading, workspaceId) {
    if (!input) {
        throw Error('No source to estimate reply');
    }
    const maxScore = 10;
    if (typeof criteria === 'string') {
        criteria = [criteria];
    }
    (0, invariant_1.default)(Array.isArray(criteria), 'G-Eval criteria is not an array');
    let gEvalPrompt = '';
    if (evalType === 'video') {
        gEvalPrompt = multiModal_1.VIDEO_GEVAL_PROMPT_EVALUATE;
    }
    const evalRubricPrompt = typeof grading?.rubricPrompt === 'object' && !Array.isArray(grading?.rubricPrompt)
        ? grading?.rubricPrompt?.['evaluate']
        : undefined;
    const evalPrompt = await loadRubricPrompt(evalRubricPrompt, gEvalPrompt);
    const promptText = await renderLlmRubricPrompt(evalPrompt, {
        steps: criteria.join('\n- '),
        maxScore: maxScore.toString(),
        input: tryParse(input),
    });
    let multimodelProvider = (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).videoProvider;
    let llmprompt = [];
    if (evalType === 'video') {
        multimodelProvider = (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).videoProvider; //TODO: Hardcoded for now
        llmprompt = [
            {
                role: 'user',
                content: [
                    { type: 'text', text: promptText },
                    { type: 'video_url', video_url: { url: output } },
                ],
            },
        ];
    }
    const resp = await multimodelProvider.callApi(JSON.stringify(llmprompt));
    let result;
    try {
        result = JSON.parse(resp.output.match(/\{.+\}/g)[0]);
    }
    catch {
        result = resp.output;
    }
    return {
        pass: result.score / maxScore >= threshold,
        score: result.score / maxScore,
        reason: result.reason,
        tokensUsed: { ...resp.tokenUsage, model: defaults_1.MULTIMODAL_MODEL_NAME },
    };
}
async function matchesAnswerRelevance(input, output, threshold, grading, workspaceId) {
    const embeddingProvider = await getAndCheckProvider('embedding', grading?.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).embeddingProvider, 'answer relevancy check');
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).gradingProvider, 'answer relevancy check');
    const tokensUsed = {
        total: 0,
        prompt: 0,
        completion: 0,
        cached: 0,
        completionDetails: {
            reasoning: 0,
            acceptedPrediction: 0,
            rejectedPrediction: 0,
        },
    };
    const candidateQuestions = [];
    for (let i = 0; i < 3; i++) {
        // TODO(ian): Parallelize
        const rubricPrompt = await loadRubricPrompt(grading?.rubricPrompt, prompts_1.ANSWER_RELEVANCY_GENERATE);
        const promptText = await renderLlmRubricPrompt(rubricPrompt, { answer: tryParse(output) });
        const resp = await textProvider.callApi(promptText);
        if (resp.error || !resp.output) {
            // Initialize a new object to avoid mutating the original
            const updatedTokensUsed = { ...tokensUsed };
            // Update with safe assignments
            updatedTokensUsed.total = (updatedTokensUsed.total || 0) + (resp.tokenUsage?.total || 0);
            updatedTokensUsed.prompt = (updatedTokensUsed.prompt || 0) + (resp.tokenUsage?.prompt || 0);
            updatedTokensUsed.completion =
                (updatedTokensUsed.completion || 0) + (resp.tokenUsage?.completion || 0);
            updatedTokensUsed.cached = (updatedTokensUsed.cached || 0) + (resp.tokenUsage?.cached || 0);
            updatedTokensUsed.model = 'text-embedding-3-small';
            // Create a new completionDetails object
            updatedTokensUsed.completionDetails = {
                reasoning: (updatedTokensUsed.completionDetails?.reasoning || 0) +
                    (resp.tokenUsage?.completionDetails?.reasoning || 0),
                acceptedPrediction: (updatedTokensUsed.completionDetails?.acceptedPrediction || 0) +
                    (resp.tokenUsage?.completionDetails?.acceptedPrediction || 0),
                rejectedPrediction: (updatedTokensUsed.completionDetails?.rejectedPrediction || 0) +
                    (resp.tokenUsage?.completionDetails?.rejectedPrediction || 0),
            };
            return fail(resp.error || 'No output', updatedTokensUsed);
        }
        (0, invariant_1.default)(typeof resp.output === 'string', 'answer relevancy check produced malformed response');
        candidateQuestions.push(resp.output);
    }
    (0, invariant_1.default)(typeof embeddingProvider.callEmbeddingApi === 'function', `Provider ${embeddingProvider.id} must implement callEmbeddingApi for similarity check`);
    const inputEmbeddingResp = await embeddingProvider.callEmbeddingApi(input);
    if (inputEmbeddingResp.error || !inputEmbeddingResp.embedding) {
        tokensUsed.total = (tokensUsed.total || 0) + (inputEmbeddingResp.tokenUsage?.total || 0);
        tokensUsed.prompt = (tokensUsed.prompt || 0) + (inputEmbeddingResp.tokenUsage?.prompt || 0);
        tokensUsed.completion =
            (tokensUsed.completion || 0) + (inputEmbeddingResp.tokenUsage?.completion || 0);
        tokensUsed.cached = (tokensUsed.cached || 0) + (inputEmbeddingResp.tokenUsage?.cached || 0);
        tokensUsed.model = 'text-embedding-3-small';
        if (tokensUsed.completionDetails && inputEmbeddingResp.tokenUsage?.completionDetails) {
            tokensUsed.completionDetails.reasoning =
                (tokensUsed.completionDetails.reasoning || 0) +
                    (inputEmbeddingResp.tokenUsage.completionDetails.reasoning || 0);
            tokensUsed.completionDetails.acceptedPrediction =
                (tokensUsed.completionDetails.acceptedPrediction || 0) +
                    (inputEmbeddingResp.tokenUsage.completionDetails.acceptedPrediction || 0);
            tokensUsed.completionDetails.rejectedPrediction =
                (tokensUsed.completionDetails.rejectedPrediction || 0) +
                    (inputEmbeddingResp.tokenUsage.completionDetails.rejectedPrediction || 0);
        }
        return fail(inputEmbeddingResp.error || 'No embedding', tokensUsed);
    }
    const inputEmbedding = inputEmbeddingResp.embedding;
    const similarities = [];
    for (const question of candidateQuestions) {
        const resp = await embeddingProvider.callEmbeddingApi(question);
        if (resp.error || !resp.embedding) {
            tokensUsed.total = (tokensUsed.total || 0) + (resp.tokenUsage?.total || 0);
            tokensUsed.prompt = (tokensUsed.prompt || 0) + (resp.tokenUsage?.prompt || 0);
            tokensUsed.completion = (tokensUsed.completion || 0) + (resp.tokenUsage?.completion || 0);
            tokensUsed.cached = (tokensUsed.cached || 0) + (resp.tokenUsage?.cached || 0);
            if (tokensUsed.completionDetails && resp.tokenUsage?.completionDetails) {
                tokensUsed.completionDetails.reasoning =
                    (tokensUsed.completionDetails.reasoning || 0) +
                        (resp.tokenUsage.completionDetails.reasoning || 0);
                tokensUsed.completionDetails.acceptedPrediction =
                    (tokensUsed.completionDetails.acceptedPrediction || 0) +
                        (resp.tokenUsage.completionDetails.acceptedPrediction || 0);
                tokensUsed.completionDetails.rejectedPrediction =
                    (tokensUsed.completionDetails.rejectedPrediction || 0) +
                        (resp.tokenUsage.completionDetails.rejectedPrediction || 0);
            }
            if (lodash_1.default.isEmpty(resp.tokenUsage?.model) === false) {
                tokensUsed.model = resp.tokenUsage?.model;
            }
            return fail(resp.error || 'No embedding', tokensUsed);
        }
        similarities.push(cosineSimilarity(inputEmbedding, resp.embedding));
    }
    const similarity = similarities.reduce((a, b) => a + b, 0) / similarities.length;
    const pass = similarity >= threshold - Number.EPSILON;
    const greaterThanReason = `Relevance ${similarity.toFixed(2)} is greater than threshold ${threshold}`;
    const lessThanReason = `Relevance ${similarity.toFixed(2)} is less than threshold ${threshold}`;
    if (pass) {
        return {
            pass: true,
            score: similarity,
            reason: greaterThanReason,
            tokensUsed,
        };
    }
    return {
        pass: false,
        score: similarity,
        reason: lessThanReason,
        tokensUsed,
    };
}
async function matchesContextRecall(context, groundTruth, threshold, grading, vars, workspaceId) {
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).gradingProvider, 'context recall check');
    const rubricPrompt = await loadRubricPrompt(grading?.rubricPrompt, prompts_1.CONTEXT_RECALL);
    const promptText = await renderLlmRubricPrompt(rubricPrompt, {
        context,
        groundTruth,
        ...(vars || {}),
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'context-recall produced malformed response');
    const sentences = splitIntoSentences(resp.output);
    const numerator = sentences.reduce((acc, sentence) => acc + (sentence.includes(prompts_1.CONTEXT_RECALL_ATTRIBUTED_TOKEN) ? 1 : 0), 0);
    const score = numerator / sentences.length;
    const pass = score >= threshold - Number.EPSILON;
    return {
        pass,
        score,
        reason: pass
            ? `Recall ${score.toFixed(2)} is >= ${threshold}`
            : `Recall ${score.toFixed(2)} is < ${threshold}`,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
            cached: resp.tokenUsage?.cached || 0,
            model: resp.tokenUsage?.model || '',
            completionDetails: resp.tokenUsage?.completionDetails || {
                reasoning: 0,
                acceptedPrediction: 0,
                rejectedPrediction: 0,
            },
        },
    };
}
async function matchesContextRelevance(question, context, threshold, grading, workspaceId) {
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).gradingProvider, 'context relevance check');
    const rubricPrompt = await loadRubricPrompt(grading?.rubricPrompt, prompts_1.CONTEXT_RELEVANCE);
    const promptText = await renderLlmRubricPrompt(rubricPrompt, {
        context,
        query: question,
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'context-relevance produced malformed response');
    const sentences = splitIntoSentences(resp.output);
    const numerator = sentences.reduce((acc, sentence) => acc + (sentence.includes(prompts_1.CONTEXT_RELEVANCE_BAD) ? 0 : 1), 0);
    const score = numerator / sentences.length;
    const pass = score >= threshold - Number.EPSILON;
    return {
        pass,
        score,
        reason: pass
            ? `Relevance ${score.toFixed(2)} is >= ${threshold}`
            : `Relevance ${score.toFixed(2)} is < ${threshold}`,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
            cached: resp.tokenUsage?.cached || 0,
            model: resp.tokenUsage?.model || '',
            completionDetails: resp.tokenUsage?.completionDetails || {
                reasoning: 0,
                acceptedPrediction: 0,
                rejectedPrediction: 0,
            },
        },
    };
}
async function matchesContextFaithfulness(query, output, context, threshold, grading, vars, workspaceId) {
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).gradingProvider, 'faithfulness check');
    if (grading?.rubricPrompt) {
        (0, invariant_1.default)(Array.isArray(grading.rubricPrompt), 'rubricPrompt must be an array');
    }
    const longformPrompt = (typeof grading?.rubricPrompt?.[0] === 'string'
        ? grading?.rubricPrompt?.[0]
        : grading?.rubricPrompt?.[0].content) || prompts_1.CONTEXT_FAITHFULNESS_LONGFORM;
    const nliPrompt = (typeof grading?.rubricPrompt?.[1] === 'string'
        ? grading?.rubricPrompt?.[1]
        : grading?.rubricPrompt?.[1].content) || prompts_1.CONTEXT_FAITHFULNESS_NLI_STATEMENTS;
    let promptText = await renderLlmRubricPrompt(longformPrompt, {
        question: query,
        answer: tryParse(output),
        ...(vars || {}),
    });
    let resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'context-faithfulness produced malformed response');
    const statements = splitIntoSentences(resp.output);
    promptText = await renderLlmRubricPrompt(nliPrompt, {
        context,
        statements,
        ...(vars || {}),
    });
    resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'context-faithfulness produced malformed response');
    let finalAnswer = 'Final verdict for each statement in order:';
    finalAnswer = finalAnswer.toLowerCase();
    let verdicts = resp.output.toLowerCase().trim();
    let score;
    if (verdicts.includes(finalAnswer)) {
        verdicts = verdicts.slice(verdicts.indexOf(finalAnswer) + finalAnswer.length);
        score =
            verdicts.split('.').filter((answer) => answer.trim() !== '' && !answer.includes('yes'))
                .length / statements.length;
    }
    else {
        score = (verdicts.split('verdict: no').length - 1) / statements.length;
    }
    score = 1 - score;
    const pass = score >= threshold - Number.EPSILON;
    return {
        pass,
        score,
        reason: pass
            ? `Faithfulness ${score.toFixed(2)} is >= ${threshold}`
            : `Faithfulness ${score.toFixed(2)} is < ${threshold}`,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
            cached: resp.tokenUsage?.cached || 0,
            model: resp.tokenUsage?.model || '',
            completionDetails: resp.tokenUsage?.completionDetails || {
                reasoning: 0,
                acceptedPrediction: 0,
                rejectedPrediction: 0,
            },
        },
    };
}
async function matchesPlanCorrectness(task, definitions, plan, grading) {
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).gradingProvider, 'plan-quality');
    const definitionsStr = typeof definitions === 'string'
        ? definitions
        : JSON.stringify(definitions ?? [], null, 2);
    const promptText = await renderLlmRubricPrompt(prompts_1.PLAN_ADHERENCE_PROMPT, {
        task,
        definitions: definitionsStr,
        plan,
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'plan-quality produced malformed response');
    const raw = resp.output.trim();
    let score;
    let pass;
    let reason;
    let parsed = null;
    try {
        parsed = (0, json_1.extractFirstJsonObject)(raw);
    }
    catch {
        // Not JSON; fall back to single-word parsing below
    }
    const verdict = parsed?.result != null
        ? String(parsed.result).trim().toLowerCase()
        : raw.toLowerCase().replace(/^["']|["']$/g, '');
    const parsedReason = parsed?.reason != null && String(parsed.reason).trim() !== ''
        ? String(parsed.reason).trim()
        : null;
    if (verdict === 'ideal') {
        score = 1;
        pass = true;
        reason =
            parsedReason ??
                'Plan is ideal: valid, uses only available tools, shortest, and will likely accomplish the task';
    }
    else if (verdict === 'valid') {
        score = 0.5;
        pass = true;
        reason =
            parsedReason ??
                'Plan is valid and uses only available tools, but has doubts on whether it can successfully accomplish the task';
    }
    else if (verdict === 'invalid') {
        score = 0;
        pass = false;
        reason =
            parsedReason ??
                'Plan includes invalid steps that cannot be used based on the available tools';
    }
    else {
        score = 0;
        pass = false;
        logger_1.default.warn(`plan-quality: unexpected evaluation response: ${raw}`);
        reason = 'Plan-correctness evaluation failed: response was not in the expected format.';
    }
    return {
        pass,
        score,
        reason,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
            cached: resp.tokenUsage?.cached || 0,
            model: resp.tokenUsage?.model || '',
            completionDetails: resp.tokenUsage?.completionDetails || {
                reasoning: 0,
                acceptedPrediction: 0,
                rejectedPrediction: 0,
            },
        },
    };
}
async function matchesPlanAdherence(task, plan, toolCalls, grading) {
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).gradingProvider, 'plan-adherence');
    const traceStr = typeof toolCalls === 'string'
        ? toolCalls
        : JSON.stringify(toolCalls ?? [], null, 2);
    const promptText = await renderLlmRubricPrompt(prompts_1.PLAN_ADHERENCE_EVAL_PROMPT, {
        task,
        plan,
        trace: traceStr,
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'plan-adherence produced malformed response');
    const raw = resp.output.trim();
    let score;
    let pass;
    let reason;
    let parsed = null;
    try {
        parsed = (0, json_1.extractFirstJsonObject)(raw);
    }
    catch {
        // Not JSON; fall back to single-word parsing below
    }
    const verdict = parsed?.result != null
        ? String(parsed.result).trim().toLowerCase()
        : raw.toLowerCase().replace(/^["']|["']$/g, '');
    const parsedReason = parsed?.reason != null && String(parsed.reason).trim() !== ''
        ? String(parsed.reason).trim()
        : null;
    // Map result to score: perfect=1.0, strong=0.75, partial=0.5, weak=0.25, none=0.0
    // Also accept numeric string (e.g. "1.0", "0.75")
    const numVal = Number(verdict);
    if (!Number.isNaN(numVal) && [0, 0.25, 0.5, 0.75, 1].includes(numVal)) {
        score = numVal;
        pass = score >= 0.5;
        reason = parsedReason ?? `Plan adherence score: ${score}`;
    }
    else if (verdict === 'perfect') {
        score = 1;
        pass = true;
        reason =
            parsedReason ??
                'Perfect adherence: every planned step present in trace in correct order, no skipped or added steps';
    }
    else if (verdict === 'strong') {
        score = 0.75;
        pass = true;
        reason =
            parsedReason ??
                'Strong adherence: all or nearly all steps executed in order; at most one minor deviation';
    }
    else if (verdict === 'partial') {
        score = 0.5;
        pass = true;
        reason =
            parsedReason ??
                'Partial adherence: some steps match but others missing, out of order, or replaced; at least one extra or ambiguous action';
    }
    else if (verdict === 'weak') {
        score = 0.25;
        pass = false;
        reason =
            parsedReason ??
                'Weak adherence: only a few plan steps in trace or multiple extraneous actions; plan structure mostly lost';
    }
    else if (verdict === 'none') {
        score = 0;
        pass = false;
        reason =
            parsedReason ??
                'No adherence: trace shows little or no resemblance to the plan';
    }
    else {
        score = 0;
        pass = false;
        logger_1.default.warn(`plan-adherence: unexpected evaluation response: ${raw}`);
        reason = 'Plan-adherence evaluation failed: response was not in the expected format.';
    }
    return {
        pass,
        score,
        reason,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
            cached: resp.tokenUsage?.cached || 0,
            model: resp.tokenUsage?.model || '',
            completionDetails: resp.tokenUsage?.completionDetails || {
                reasoning: 0,
                acceptedPrediction: 0,
                rejectedPrediction: 0,
            },
        },
    };
}
async function matchesGoalFulfillment(task, plan, trace, agentOutput, grading) {
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).gradingProvider, 'goal-fulfillment');
    const planForPrompt = typeof plan === 'string' && plan.trim().length > 0
        ? plan
        : '(No plan was provided. Evaluate goal fulfillment based on the execution trace and agent output only.)';
    const traceStr = typeof trace === 'string' ? trace : JSON.stringify(trace ?? [], null, 2);
    const promptText = await renderLlmRubricPrompt(prompts_1.GOAL_FULFILLMENT_PROMPT, {
        task,
        plan: planForPrompt,
        trace: traceStr,
        agent_output: agentOutput,
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'goal-fulfillment produced malformed response');
    const raw = resp.output.trim();
    let score;
    let pass;
    let reason;
    let parsed = null;
    try {
        parsed = (0, json_1.extractFirstJsonObject)(raw);
    }
    catch {
        // Not JSON; fall back to single-word parsing below
    }
    const verdict = parsed?.result != null
        ? String(parsed.result).trim().toLowerCase()
        : raw.toLowerCase().replace(/^["']|["']$/g, '');
    const parsedReason = parsed?.reason != null && String(parsed.reason).trim() !== ''
        ? String(parsed.reason).trim()
        : null;
    if (verdict === 'fully_fulfilled') {
        score = 1;
        pass = true;
        reason =
            parsedReason ??
                'The agent\'s output completely and accurately addresses the user\'s goal';
    }
    else if (verdict === 'partially_fulfilled') {
        score = 0.5;
        pass = true;
        reason =
            parsedReason ??
                'The agent\'s output addresses some aspects of the goal but is incomplete or has minor inaccuracies';
    }
    else if (verdict === 'not_fulfilled') {
        score = 0;
        pass = false;
        reason =
            parsedReason ??
                'The agent\'s output fails to address the user\'s goal or is irrelevant/incorrect';
    }
    else {
        score = 0;
        pass = false;
        logger_1.default.warn(`goal-fulfillment: unexpected evaluation response: ${raw}`);
        reason = 'Goal-fulfillment evaluation failed: response was not in the expected format.';
    }
    return {
        pass,
        score,
        reason,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
            cached: resp.tokenUsage?.cached || 0,
            model: resp.tokenUsage?.model || '',
            completionDetails: resp.tokenUsage?.completionDetails || {
                reasoning: 0,
                acceptedPrediction: 0,
                rejectedPrediction: 0,
            },
        },
    };
}
async function matchesToolSelectionQuality(task, definitions, trace, grading) {
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).gradingProvider, 'tool-selection-quality');
    const availableToolsStr = typeof definitions === 'string'
        ? definitions
        : JSON.stringify(definitions ?? [], null, 2);
    const toolSelectionStr = typeof trace === 'string' ? trace : JSON.stringify(trace ?? [], null, 2);
    const promptText = await renderLlmRubricPrompt(prompts_1.TOOL_SELECTION_QUALITY_PROMPT, {
        input: task,
        available_tools: availableToolsStr,
        tool_selection: toolSelectionStr,
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'tool-selection-quality produced malformed response');
    const raw = resp.output.trim();
    let score;
    let pass;
    let reason;
    let parsed = null;
    try {
        parsed = (0, json_1.extractFirstJsonObject)(raw);
    }
    catch {
        // Not JSON; fall back to single-word parsing below
    }
    const verdict = parsed?.result != null
        ? String(parsed.result).trim().toLowerCase()
        : raw.toLowerCase().replace(/^["']|["']$/g, '');
    const parsedReason = parsed?.reason != null && String(parsed.reason).trim() !== ''
        ? String(parsed.reason).trim()
        : null;
    if (verdict === 'correct') {
        score = 1;
        pass = true;
        reason =
            parsedReason ??
                'Tool selection was correct: best available tool(s) chosen and within allowed set.';
    }
    else if (verdict === 'incorrect') {
        score = 0;
        pass = false;
        reason =
            parsedReason ??
                'Tool selection was incorrect based on the evaluation criteria.';
    }
    else {
        score = 0;
        pass = false;
        logger_1.default.warn(`tool-selection-quality: unexpected evaluation response: ${raw}`);
        reason =
            'Tool-selection-quality evaluation failed: response was not in the expected format.';
    }
    return {
        pass,
        score,
        reason,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
            cached: resp.tokenUsage?.cached || 0,
            model: resp.tokenUsage?.model || '',
            completionDetails: resp.tokenUsage?.completionDetails || {
                reasoning: 0,
                acceptedPrediction: 0,
                rejectedPrediction: 0,
            },
        },
    };
}
async function matchesExecutionEfficiency(task, trace, agentOutput, grading) {
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).gradingProvider, 'execution-efficiency');
    const traceStr = typeof trace === 'string' ? trace : JSON.stringify(trace ?? [], null, 2);
    const promptText = await renderLlmRubricPrompt(prompts_1.EXECUTION_EFFICIENCY_PROMPT, {
        task,
        trace: traceStr,
        agent_output: agentOutput,
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'execution-efficiency produced malformed response');
    const raw = resp.output.trim();
    let score;
    let pass;
    let reason;
    let parsed = null;
    try {
        parsed = (0, json_1.extractFirstJsonObject)(raw);
    }
    catch {
        // Not JSON; fall back to parsing a bare integer
    }
    const parsedReason = parsed?.reason != null && String(parsed.reason).trim() !== ''
        ? String(parsed.reason).trim()
        : null;
    const rawScore = parsed?.score != null ? Number(parsed.score) : Number(raw.trim());
    if (!Number.isNaN(rawScore) && rawScore >= 0 && rawScore <= 3) {
        score = rawScore / 3;
        pass = rawScore >= 2;
        reason = parsedReason ?? `Execution efficiency score: ${rawScore}/3`;
    }
    else {
        score = 0;
        pass = false;
        logger_1.default.warn(`execution-efficiency: unexpected evaluation response: ${raw}`);
        reason = 'Execution-efficiency evaluation failed: response was not in the expected format.';
    }
    return {
        pass,
        score,
        reason,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
            cached: resp.tokenUsage?.cached || 0,
            model: resp.tokenUsage?.model || '',
            completionDetails: resp.tokenUsage?.completionDetails || {
                reasoning: 0,
                acceptedPrediction: 0,
                rejectedPrediction: 0,
            },
        },
    };
}
async function matchesReasoningCoherence(task, plan, trace, agentOutput, grading) {
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).gradingProvider, 'reasoning-coherence');
    const traceStr = typeof trace === 'string' ? trace : JSON.stringify(trace ?? [], null, 2);
    const planForPrompt = typeof plan === 'string' && plan.trim().length > 0
        ? plan
        : '(No plan was provided. Evaluate the logical consistency of the execution trace and agent output only.)';
    const promptText = await renderLlmRubricPrompt(prompts_1.REASONING_COHERENCE_PROMPT, {
        task,
        plan: planForPrompt,
        trace: traceStr,
        agent_output: agentOutput,
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'reasoning-coherence produced malformed response');
    const raw = resp.output.trim();
    let score;
    let pass;
    let reason;
    let parsed = null;
    try {
        parsed = (0, json_1.extractFirstJsonObject)(raw);
    }
    catch {
        // Not JSON; fall back to parsing a bare integer
    }
    const parsedReason = parsed?.reason != null && String(parsed.reason).trim() !== ''
        ? String(parsed.reason).trim()
        : null;
    const rawScore = parsed?.score != null ? Number(parsed.score) : Number(raw.trim());
    if (!Number.isNaN(rawScore) && rawScore >= 0 && rawScore <= 3) {
        score = rawScore / 3;
        pass = rawScore >= 2;
        reason = parsedReason ?? `Reasoning coherence score: ${rawScore}/3`;
    }
    else {
        score = 0;
        pass = false;
        logger_1.default.warn(`reasoning-coherence: unexpected evaluation response: ${raw}`);
        reason =
            'Reasoning-coherence evaluation failed: response was not in the expected format.';
    }
    return {
        pass,
        score,
        reason,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
            cached: resp.tokenUsage?.cached || 0,
            model: resp.tokenUsage?.model || '',
            completionDetails: resp.tokenUsage?.completionDetails || {
                reasoning: 0,
                acceptedPrediction: 0,
                rejectedPrediction: 0,
            },
        },
    };
}
async function matchesAgentCustomEval(task, plan, definitions, trace, agentOutput, criteria, grading, threshold = 0.7) {
    const maxScore = Math.max(1, criteria.length);
    const criteriaStr = criteria.map((c, i) => `${i + 1}. ${c}`).join('\n');
    const traceStr = typeof trace === 'string' ? trace : JSON.stringify(trace ?? [], null, 2);
    const definitionsStr = typeof definitions === 'string' ? definitions : JSON.stringify(definitions ?? [], null, 2);
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).gradingProvider, 'agent-custom-eval');
    const promptText = await renderLlmRubricPrompt(prompts_1.AGENT_CUSTOM_EVAL_PROMPT, {
        task,
        plan,
        definitions: definitionsStr,
        trace: traceStr,
        agent_output: agentOutput,
        criteria: criteriaStr,
        max_score: maxScore.toString(),
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'agent-custom-eval produced malformed response');
    const raw = resp.output.trim();
    let score;
    let pass;
    let reason;
    let parsed = null;
    try {
        parsed = (0, json_1.extractFirstJsonObject)(raw);
    }
    catch {
        // Not JSON; fall back to parsing a bare integer
    }
    const parsedReason = parsed?.reason != null && String(parsed.reason).trim() !== ''
        ? String(parsed.reason).trim()
        : null;
    const rawScore = parsed?.score != null ? Number(parsed.score) : Number(raw.trim());
    if (!Number.isNaN(rawScore) && rawScore >= 0 && rawScore <= maxScore) {
        score = rawScore / maxScore;
        pass = score >= threshold;
        reason = parsedReason ?? `Agent custom eval score: ${rawScore}/${maxScore}`;
    }
    else {
        score = 0;
        pass = false;
        logger_1.default.warn(`agent-custom-eval: unexpected evaluation response: ${raw}`);
        reason =
            'Agent-custom-eval evaluation failed: response was not in the expected format.';
    }
    return {
        pass,
        score,
        reason,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
            cached: resp.tokenUsage?.cached || 0,
            model: resp.tokenUsage?.model || '',
            completionDetails: resp.tokenUsage?.completionDetails || {
                reasoning: 0,
                acceptedPrediction: 0,
                rejectedPrediction: 0,
            },
        },
    };
}
async function matchesSelectBest(criteria, outputs, grading, vars, workspaceId) {
    (0, invariant_1.default)(outputs.length >= 2, 'select-best assertion must have at least two outputs to compare between');
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).gradingProvider, 'select-best check');
    const rubricPrompt = await loadRubricPrompt(grading?.rubricPrompt, prompts_1.SELECT_BEST_PROMPT);
    const promptText = await renderLlmRubricPrompt(rubricPrompt, {
        criteria,
        outputs: outputs.map((o) => tryParse(o)),
        ...(vars || {}),
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return new Array(outputs.length).fill(fail(resp.error || 'No output', resp.tokenUsage));
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'select-best produced malformed response');
    const firstDigitMatch = resp.output.trim().match(/\d/);
    const verdict = firstDigitMatch ? Number.parseInt(firstDigitMatch[0], 10) : Number.NaN;
    if (Number.isNaN(verdict) || verdict < 0 || verdict >= outputs.length) {
        return new Array(outputs.length).fill(fail(`Invalid select-best verdict: ${verdict}`));
    }
    const tokensUsed = {
        total: resp.tokenUsage?.total || 0,
        prompt: resp.tokenUsage?.prompt || 0,
        completion: resp.tokenUsage?.completion || 0,
        cached: resp.tokenUsage?.cached || 0,
        model: resp.tokenUsage?.model || '',
        completionDetails: resp.tokenUsage?.completionDetails || {
            reasoning: 0,
            acceptedPrediction: 0,
            rejectedPrediction: 0,
        },
    };
    return outputs.map((output, index) => {
        if (index === verdict) {
            return {
                pass: true,
                score: 1,
                reason: `Output selected as the best: ${criteria}`,
                tokensUsed,
            };
        }
        else {
            return {
                pass: false,
                score: 0,
                reason: `Output not selected: ${criteria}`,
                tokensUsed,
            };
        }
    });
}
async function matchesModeration({ userPrompt, assistantResponse, categories = [] }, grading, workspaceId) {
    if (!assistantResponse) {
        return {
            pass: true,
            score: 1,
            reason: 'No output to moderate',
        };
    }
    // Get default providers
    const defaultProviders = await (0, defaults_1.getDefaultProviders)(undefined, workspaceId);
    // Only try to use Replicate if OpenAI is not available
    const hasOpenAiKey = (0, envars_1.getEnvString)('OPENAI_API_KEY');
    const hasReplicateKey = !hasOpenAiKey && ((0, envars_1.getEnvString)('REPLICATE_API_KEY') || (0, envars_1.getEnvString)('REPLICATE_API_TOKEN'));
    const defaultModerationProvider = hasReplicateKey
        ? await (0, providers_1.loadApiProvider)(constants_1.LLAMA_GUARD_REPLICATE_PROVIDER)
        : defaultProviders.moderationProvider;
    const moderationProvider = (await getAndCheckProvider('moderation', grading?.provider, defaultModerationProvider, 'moderation check'));
    (0, invariant_1.default)(moderationProvider, 'Moderation provider must be defined');
    const resp = await moderationProvider.callModerationApi(userPrompt, assistantResponse);
    if (resp.error) {
        return {
            pass: false,
            score: 0,
            reason: `Moderation API error: ${resp.error}`,
        };
    }
    const { flags } = resp;
    if (!flags || flags.length === 0) {
        return {
            pass: true,
            score: 1,
            reason: 'No moderation flags detected',
        };
    }
    const filteredFlags = categories.length === 0 ? flags : flags.filter((flag) => categories.includes(flag.code));
    if (filteredFlags.length > 0) {
        return {
            pass: false,
            score: 0,
            reason: `Moderation flags detected: ${filteredFlags
                .map((flag) => flag.description)
                .join(', ')}`,
        };
    }
    return {
        pass: true,
        score: 1,
        reason: 'No relevant moderation flags detected',
    };
}
async function matchesSentiment(criteria, input, output, grading, workspaceId) {
    if (!input) {
        throw Error('No assertion text to estimate reply');
    }
    const textProvider = (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).audioProvider; //TODO: Hardcoded for now
    const promptSteps = (0, dedent_1.default) `
    You are a helpful AI bot that checks for the tone of the audio.
    Analyze the audio file and determine the tone (e.g., positive, neutral, negative).
    Your evaluation should provide a multiclass label from the following options: ['positive', 'neutral', 'negative'].
    IMPORTANT: Respond only with one word. Your response must be a string, either positive, neutral, or negative, and should not contain any text or characters aside from that.
    `;
    const prompt = [
        {
            role: 'user',
            content: [
                { type: 'text', text: promptSteps },
                { type: 'input_audio', input_audio: { data: output, format: 'wav' } },
            ],
        },
    ];
    const promptText = JSON.stringify(prompt);
    const resp = await textProvider.callApi(promptText);
    const result = resp.output.toLowerCase();
    const pass = criteria.includes(result);
    return {
        pass,
        score: pass ? 1 : 0,
        reason: `The sentiment of the audio was deemed to be '${result}'`,
        tokensUsed: resp.tokenUsage,
    };
}
const CODE_CORRECTNESS_PROMPT = (0, dedent_1.default) `
Code Correctness: First Identify whether the provided text contains code and then evaluate the provided code to determine its correctness in solving the given
instruction.

Data:
-----
[Instruction]: {{ input }}
Clearly define the task or problem that the code aims to address.

[Reference Code]: {{ output }}
Examine the submitted code for evaluation in the context of the provided instruction. 
This code was written by an LLM (AI Bot Agent).

Evaluation:
-----------
Please read the instruction and the LLM's generated code carefully. Your task is to determine if the code correctly and efficiently solves the instruction with no bugs.

Return **ONLY a valid JSON object** with the following two fields:
- "label": A single word, either "correct" or "incorrect".
- "explanation": Your step-by-step reasoning for the verdict.

**Example response:**
\`\`\`json
{
  "label": "correct",
  "explanation": "The Fibonacci function is correctly implemented using recursion, handling base cases and the recursive step appropriately. The loop prints the first 10 Fibonacci numbers as expected."
}
\`\`\`

Your entire response **MUST BE a valid JSON object**, exactly like the example above, and contain **NO other text or formatting** outside the JSON. Do not include any introductory or concluding remarks.

Note: Assumptions can be made that any code needed for the instruction is correct, and optimization
is not a requirement for a correct solution.
`;
const CONTAINS_CODE_LANGUAGE_PROMPT = (0, dedent_1.default) `
  Contains Code Language: Evaluate the provided text below to determine if it contains code written in the programming language – {{ language }}.

  Data:
  -----
  [text]: {{ output }}

  Evaluation:
  -----------
  Provide a concise response in a json format with a field called “result” with the value of “yes” or “no” depending on whether the programming language was detected or not. Also, add a field in the json response called “language” which has the value of actual programming language detected. The language field is an array with value as all languages detected.
  Example:
  -----------

  [code]: 'def fibonacci(n):\n    if n <= 1:\n        return n\n    else:\n        return
  fibonacci(n - 1) + fibonacci(n - 2)\n\nfor i in range(10):\n    print(fibonacci(i))'

  Example response:
  ************
  {
  	“language”: [“python”],
  	result: “yes”
  }
  ************
  `;
async function matchesCodeCorrectness(prompt, output, grading, workspaceId) {
    if (!grading) {
        throw new Error('Grading config is required for code correctness check.');
    }
    const textProvider = await getAndCheckProvider('text', grading.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).gradingProvider, 'code-correctness check');
    const promptText = nunjucks.renderString(CODE_CORRECTNESS_PROMPT, {
        input: prompt,
        output, // Using shorthand here
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        let errorMessage = 'No response from model';
        if (resp.error) {
            if (typeof resp.error === 'string') {
                errorMessage = resp.error;
            }
            else if (typeof resp.error === 'object' && resp.error !== null && 'message' in resp.error) {
                errorMessage = resp.error.message;
            }
            else {
                errorMessage = JSON.stringify(resp.error);
            }
        }
        return {
            pass: false,
            score: 0,
            reason: errorMessage,
            tokensUsed: resp.tokenUsage,
        };
    }
    try {
        // Current parsing logic for matchesCodeCorrectness (extracts JSON string, then parses)
        const jsonString = resp.output.trim().match(/\{[\s\S]*?\}/)?.[0];
        if (!jsonString) {
            return {
                pass: false,
                score: 0,
                reason: `No valid JSON found in response: ${resp.output}`,
                tokensUsed: resp.tokenUsage,
            };
        }
        const parsed = JSON.parse(jsonString);
        const { label, explanation } = parsed; // Destructuring 'label' and 'explanation'
        const pass = label?.toLowerCase() === 'correct';
        const score = pass ? 1 : 0;
        return {
            pass,
            score,
            // --- MODIFICATION HERE: Construct the 'reason' string to match the desired format ---
            reason: `${label}\n${explanation}`,
            // ----------------------------------------------------------------------------------
            tokensUsed: resp.tokenUsage,
        };
    }
    catch (err) {
        return {
            pass: false,
            score: 0,
            reason: `Failed to parse JSON: ${err.message}`,
            tokensUsed: resp.tokenUsage,
        };
    }
}
exports.CODE_LANGUAGE_PROMPT = `
Code Language Detection:

Task:
Identify whether the provided text contains code and, if so, which programming language(s) 
it is written in.

Input:
[text]: {{ output }}

Instructions:
Return a JSON object with:
- "result": "yes" or "no" — depending on whether code is present
- "language": an array of detected languages (like ["python", "javascript"])

Example Response:
{
  "result": "yes",
  "language": ["python"]
}
`;
async function detectCodeLanguage(output, grading, promptLanguages, workspaceId) {
    if (!grading) {
        throw new Error('Grading config is required to detect code language.');
    }
    const textProvider = await getAndCheckProvider('text', grading.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).gradingProvider, 'code-language check');
    const promptText = nunjucks.renderString(exports.CODE_LANGUAGE_PROMPT, {
        output: JSON.stringify(output).slice(1, -1),
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return {
            pass: false,
            score: 0,
            reason: resp.error || 'No response from model',
            tokensUsed: resp.tokenUsage,
        };
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'Response must be a string');
    try {
        const jsonString = resp.output.trim().match(/\{[\s\S]*?\}/)?.[0];
        if (!jsonString) {
            return {
                pass: false,
                score: 0,
                reason: `No valid JSON found in response: ${resp.output}`,
                tokensUsed: resp.tokenUsage,
            };
        }
        const parsed = JSON.parse(jsonString);
        const { result, language } = parsed;
        const languages = Array.isArray(language)
            ? language.map((lang) => lang.toLowerCase())
            : [];
        const promptLanguagesArray = promptLanguages ? promptLanguages.split(',') : [];
        const pass = result?.toLowerCase() === 'yes' &&
            promptLanguagesArray.every((lang) => languages.includes(lang.toLowerCase()));
        const score = pass ? 1 : 0;
        const reason = result?.toLowerCase() === 'yes'
            ? `Detected Code Language: ${languages.join(', ')}`
            : `No code detected in output.`;
        return {
            pass,
            score,
            reason,
            tokensUsed: resp.tokenUsage,
            rawOutput: resp.output,
            language: languages,
        };
    }
    catch (err) {
        return {
            pass: false,
            score: 0,
            reason: `Failed to parse JSON: ${err.message}`,
            tokensUsed: resp.tokenUsage,
        };
    }
}
/**
 * Evaluates if the provided text contains code written in a specific programming language.
 * Uses an LLM to detect the language(s) and presence of code.
 */
async function matchesContainsCodeLanguage(language, // The expected programming language to look for.
output, // The text to be evaluated for code presence.
grading, // Optional grading configuration, including the grader provider.
vars, // Additional variables for prompt templating.
workspaceId) {
    if (!grading) {
        throw new Error('Cannot grade output without grading config. Specify --grader option or grading config.');
    }
    const textProvider = await getAndCheckProvider('text', grading.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).gradingProvider, 'contains code language check');
    const rubricPrompt = await loadRubricPrompt(grading?.rubricPrompt, CONTAINS_CODE_LANGUAGE_PROMPT);
    const prompt = nunjucks.renderString(rubricPrompt, {
        language: JSON.stringify(language).slice(1, -1), // Expected language to detect
        output: JSON.stringify(output).slice(1, -1), // Text containing code
        ...fromVars(vars),
    });
    const resp = await textProvider.callApi(prompt);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output from contains code language checker', resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'Contains code language checker produced malformed response');
    try {
        const jsonOutput = resp.output.trim().match(/\{.+\}/s)?.[0]; // Use /s for dotall to match newlines
        if (!jsonOutput) {
            return fail(`Contains code language checker output did not contain valid JSON: ${resp.output}`, resp.tokenUsage);
        }
        const result = JSON.parse(jsonOutput);
        // Validate JSON structure
        if (typeof result.result !== 'string' || !Array.isArray(result.language)) {
            return fail(`Malformed JSON response from contains code language checker: ${resp.output}`, resp.tokenUsage);
        }
        const detectedResult = result.result.toLowerCase();
        const detectedLanguages = result.language.map((lang) => lang.toLowerCase());
        const pass = detectedResult === 'yes' && detectedLanguages.includes(language.toLowerCase());
        let reason;
        if (pass) {
            reason = `Text contains code in "${language}". Detected languages: ${detectedLanguages.join(', ')}.`;
        }
        else if (detectedResult === 'no') {
            reason = `Text does not contain code in "${language}". No code detected.`;
        }
        else {
            // detectedResult is 'yes' but 'language' not found
            reason = `Text contains code, but not in "${language}". Detected languages: ${detectedLanguages.join(', ')}.`;
        }
        return {
            pass,
            score: pass ? 1 : 0,
            reason,
            tokensUsed: {
                total: resp.tokenUsage?.total || 0,
                prompt: resp.tokenUsage?.prompt || 0,
                completion: resp.tokenUsage?.completion || 0,
                cached: resp.tokenUsage?.cached || 0,
                model: resp.tokenUsage?.model || '',
                completionDetails: resp.tokenUsage?.completionDetails || {
                    reasoning: 0,
                    acceptedPrediction: 0,
                    rejectedPrediction: 0,
                },
            },
        };
    }
    catch (err) {
        return fail(`Error parsing contains code language output: ${err.message}`, resp.tokenUsage);
    }
}
// --- CODE_READABILITY_PROMPT (No change needed here, it's already good for JSON output) ---
const CODE_READABILITY_PROMPT = `
Code Readability: You are a stern but practical senior software engineer who cares a lot about simplicity
and readability of code.Identify whether the provided text contains code and Can you review the following code that was written by an LLM (AI Bot Agent)?
Focus on readability of the code. The implementation is "readable" if you think the code is readable, or
"unreadable" if the code is unreadable or needlessly complex for what it's trying to accomplish.

Task Assignment:
\`\`\`
{{ input }}
\`\`\`

Implementation to Evaluate:
\`\`\`
{{ output }}
\`\`\`

Please read the code carefully, then **return ONLY a valid JSON object**.

**Example response:**
\`\`\`json
{
  "label": "readable",
  "explanation": "The code is well-structured with clear variable names and comments. It effectively addresses the task assignment without unnecessary complexity."
}
\`\`\`

Your entire response **MUST BE a valid JSON object** and contain **NO other text or formatting**. Do not include any introductory or concluding remarks.
`;
// --- matchesCodeReadability function (Changes applied here) ---
async function matchesCodeReadability(input, // Task assignment / description
output, // Code implementation to evaluate
grading, // Grading config, must include 'provider' at least
vars, // Optional template vars
workspaceId) {
    if (!grading) {
        throw new Error('Cannot grade output without grading config. Specify --grader option or grading config.');
    }
    const defaultProviders = await (0, defaults_1.getDefaultProviders)(undefined, workspaceId);
    const textProvider = await getAndCheckProvider('text', grading.provider, defaultProviders.gradingProvider || null, 'code readability check');
    const rubricPrompt = await loadRubricPrompt(grading?.rubricPrompt, CODE_READABILITY_PROMPT);
    const prompt = nunjucks.renderString(rubricPrompt, {
        input: JSON.stringify(input).slice(1, -1),
        output: JSON.stringify(output).slice(1, -1),
        ...vars,
    });
    const resp = await textProvider.callApi(prompt);
    if (resp.error || !resp.output) {
        let errorMessage = 'No output from code readability checker';
        if (resp.error) {
            if (typeof resp.error === 'string') {
                errorMessage = resp.error;
            }
            else if (typeof resp.error === 'object' && resp.error !== null && 'message' in resp.error) {
                errorMessage = resp.error.message;
            }
            else {
                errorMessage = JSON.stringify(resp.error);
            }
        }
        return fail(errorMessage, resp.tokenUsage);
    }
    (0, invariant_1.default)(typeof resp.output === 'string', 'Code readability checker produced malformed response');
    try {
        const outputText = resp.output.trim();
        const parsedResult = (0, json_1.extractJsonObjects)(outputText)?.[0];
        if (typeof parsedResult !== 'object' || parsedResult === null) {
            return fail(`Code readability checker output did not contain a valid JSON object: ${outputText}`, resp.tokenUsage);
        }
        const label = parsedResult.label?.toLowerCase();
        const explanation = parsedResult.explanation || '';
        const pass = label === 'readable';
        return {
            pass,
            score: pass ? 1 : 0,
            // --- IMPORTANT CHANGE HERE: CONSTRUCT THE REASON STRING ---
            reason: `${label}\n${explanation}`,
            // --------------------------------------------------------
            tokensUsed: {
                total: resp.tokenUsage?.total || 0,
                prompt: resp.tokenUsage?.prompt || 0,
                completion: resp.tokenUsage?.completion || 0,
                cached: resp.tokenUsage?.cached || 0,
                model: resp.tokenUsage?.model || '',
                completionDetails: {
                    reasoning: 0,
                    acceptedPrediction: 0,
                    rejectedPrediction: 0,
                },
            },
        };
    }
    catch (err) {
        return fail(`Error parsing code readability output: ${err.message}`, resp.tokenUsage);
    }
}
// --- handleCodeReadability function (No change needed here, it correctly passes values) ---
const handleCodeReadability = async ({ assertion, prompt, test, outputString, // Use outputString as the LLM's generated code
 }) => {
    let codeToEvaluate = '';
    if (typeof outputString === 'string') {
        codeToEvaluate = outputString;
    }
    else if (outputString != null) {
        try {
            codeToEvaluate = JSON.stringify(outputString, null, 2);
        }
        catch {
            codeToEvaluate = String(outputString);
        }
    }
    (0, invariant_1.default)(typeof prompt === 'string', 'Prompt must be a string');
    if (codeToEvaluate.trim() === '') {
        return {
            assertion,
            pass: false,
            score: 0,
            reason: 'No generated code available to evaluate for readability.',
            tokensUsed: {
                total: 0,
                prompt: 0,
                completion: 0,
                cached: 0,
                model: '',
                completionDetails: {
                    reasoning: 0,
                    acceptedPrediction: 0,
                    rejectedPrediction: 0,
                },
            },
        };
    }
    const options = test?.options ?? {};
    const resp = await matchesCodeReadability(prompt, codeToEvaluate, options);
    return {
        assertion,
        ...resp,
    };
};
exports.handleCodeReadability = handleCodeReadability;
//# sourceMappingURL=matchers.js.map