"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.matchesModeration = exports.matchesSelectBest = exports.matchesContextFaithfulness = exports.matchesContextRelevance = exports.matchesContextRecall = exports.matchesAnswerRelevance = exports.matchesClosedQa = exports.matchesFactuality = exports.matchesLlmRubric = exports.matchesClassification = exports.matchesSimilarity = exports.getAndCheckProvider = exports.getGradingProvider = void 0;
const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
const logger_1 = __importDefault(require("./logger"));
const prompts_1 = require("./prompts");
const providers_1 = require("./providers");
const defaults_1 = require("./providers/defaults");
const util_1 = require("./util");
const nunjucks = (0, util_1.getNunjucksEngine)();
function cosineSimilarity(vecA, vecB) {
    if (vecA.length !== vecB.length) {
        throw new Error('Vectors must be of equal length');
    }
    const dotProduct = vecA.reduce((acc, val, idx) => acc + val * vecB[idx], 0);
    const vecAMagnitude = Math.sqrt(vecA.reduce((acc, val) => acc + val * val, 0));
    const vecBMagnitude = Math.sqrt(vecB.reduce((acc, val) => acc + val * val, 0));
    return dotProduct / (vecAMagnitude * vecBMagnitude);
}
function fromVars(vars) {
    if (!vars) {
        return {};
    }
    const ret = {};
    for (const [key, value] of Object.entries(vars)) {
        if (typeof value === 'object') {
            ret[key] = JSON.stringify(value);
        }
        else {
            ret[key] = JSON.stringify(value).slice(1, -1);
        }
    }
    return ret;
}
async function loadFromProviderOptions(provider) {
    (0, tiny_invariant_1.default)(typeof provider === 'object', `Provider must be an object, but received a ${typeof provider}: ${provider}`);
    (0, tiny_invariant_1.default)(!Array.isArray(provider), `Provider must be an object, but received an array: ${JSON.stringify(provider)}`);
    (0, tiny_invariant_1.default)(provider.id, 'Provider supplied to assertion must have an id');
    // TODO(ian): set basepath if invoked from filesystem config
    return (0, providers_1.loadApiProvider)(provider.id, { options: provider });
}
async function getGradingProvider(type, provider, defaultProvider) {
    let finalProvider;
    if (typeof provider === 'string') {
        // Defined as a string
        finalProvider = await (0, providers_1.loadApiProvider)(provider);
    }
    else if (typeof provider === 'object' && typeof provider.id === 'function') {
        // Defined as an ApiProvider interface
        finalProvider = provider;
    }
    else if (typeof provider === 'object') {
        const typeValue = provider[type];
        if (typeValue) {
            // Defined as embedding, classification, or text record
            finalProvider = await getGradingProvider(type, typeValue, defaultProvider);
        }
        else if (provider.id) {
            // Defined as ProviderOptions
            finalProvider = await loadFromProviderOptions(provider);
        }
        else if (Array.isArray(provider)) {
            throw new Error(`Provider must be an object or string, but received an array.\n\nCheck that the provider ${JSON.stringify(provider[0], null, 2)} is not nested in an array.`);
        }
        else {
            throw new Error(`Invalid provider definition for output type '${type}': ${JSON.stringify(provider, null, 2)}`);
        }
    }
    else {
        finalProvider = defaultProvider;
    }
    return finalProvider;
}
exports.getGradingProvider = getGradingProvider;
async function getAndCheckProvider(type, provider, defaultProvider, checkName) {
    const matchedProvider = await getGradingProvider(type, provider, defaultProvider);
    if (!matchedProvider) {
        if (defaultProvider) {
            logger_1.default.warn(`No provider of type ${type} found for '${checkName}', falling back to default`);
            return defaultProvider;
        }
        else {
            throw new Error(`No provider of type ${type} found for '${checkName}'`);
        }
    }
    let isValidProviderType = true;
    if (type === 'embedding') {
        isValidProviderType =
            'callEmbeddingApi' in matchedProvider || 'callSimilarityApi' in matchedProvider;
    }
    else if (type === 'classification') {
        isValidProviderType = 'callClassificationApi' in matchedProvider;
    }
    else if (type === 'moderation') {
        isValidProviderType = 'callModerationApi' in matchedProvider;
    }
    if (!isValidProviderType) {
        if (defaultProvider) {
            logger_1.default.warn(`Provider ${matchedProvider.id()} is not a valid ${type} provider for '${checkName}', falling back to default`);
            return defaultProvider;
        }
        else {
            throw new Error(`Provider ${matchedProvider.id()} is not a valid ${type} provider for '${checkName}'`);
        }
    }
    return matchedProvider;
}
exports.getAndCheckProvider = getAndCheckProvider;
function fail(reason, tokensUsed) {
    return {
        pass: false,
        score: 0,
        reason,
        tokensUsed: {
            total: tokensUsed?.total || 0,
            prompt: tokensUsed?.prompt || 0,
            completion: tokensUsed?.completion || 0,
        },
    };
}
async function matchesSimilarity(expected, output, threshold, inverse = false, grading) {
    const finalProvider = (await getAndCheckProvider('embedding', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).embeddingProvider, 'similarity check'));
    let similarity;
    let tokensUsed = {
        total: 0,
        prompt: 0,
        completion: 0,
    };
    if ('callSimilarityApi' in finalProvider) {
        const similarityResp = await finalProvider.callSimilarityApi(expected, output);
        tokensUsed = {
            ...tokensUsed,
            ...similarityResp.tokenUsage,
        };
        if (similarityResp.error) {
            return fail(similarityResp.error, tokensUsed);
        }
        if (similarityResp.similarity == null) {
            return fail('Unknown error fetching similarity', tokensUsed);
        }
        similarity = similarityResp.similarity;
    }
    else if ('callEmbeddingApi' in finalProvider) {
        const expectedEmbedding = await finalProvider.callEmbeddingApi(expected);
        const outputEmbedding = await finalProvider.callEmbeddingApi(output);
        tokensUsed = {
            total: (expectedEmbedding.tokenUsage?.total || 0) + (outputEmbedding.tokenUsage?.total || 0),
            prompt: (expectedEmbedding.tokenUsage?.prompt || 0) + (outputEmbedding.tokenUsage?.prompt || 0),
            completion: (expectedEmbedding.tokenUsage?.completion || 0) +
                (outputEmbedding.tokenUsage?.completion || 0),
        };
        if (expectedEmbedding.error || outputEmbedding.error) {
            return fail(expectedEmbedding.error || outputEmbedding.error || 'Unknown error fetching embeddings', tokensUsed);
        }
        if (!expectedEmbedding.embedding || !outputEmbedding.embedding) {
            return fail('Embedding not found', tokensUsed);
        }
        similarity = cosineSimilarity(expectedEmbedding.embedding, outputEmbedding.embedding);
    }
    else {
        throw new Error('Provider must implement callSimilarityApi or callEmbeddingApi');
    }
    const pass = inverse ? similarity <= threshold : similarity >= threshold;
    const greaterThanReason = `Similarity ${similarity.toFixed(2)} is greater than threshold ${threshold}`;
    const lessThanReason = `Similarity ${similarity.toFixed(2)} is less than threshold ${threshold}`;
    if (pass) {
        return {
            pass: true,
            score: inverse ? 1 - similarity : similarity,
            reason: inverse ? lessThanReason : greaterThanReason,
            tokensUsed,
        };
    }
    return {
        pass: false,
        score: inverse ? 1 - similarity : similarity,
        reason: inverse ? greaterThanReason : lessThanReason,
        tokensUsed,
    };
}
exports.matchesSimilarity = matchesSimilarity;
/**
 *
 * @param expected Expected classification. If undefined, matches any classification.
 * @param output Text to classify.
 * @param threshold Value between 0 and 1. If the expected classification is undefined, the threshold is the minimum score for any classification. If the expected classification is defined, the threshold is the minimum score for that classification.
 * @param grading
 * @returns Pass if the output matches the classification with a score greater than or equal to the threshold.
 */
async function matchesClassification(expected, output, threshold, grading) {
    const finalProvider = (await getAndCheckProvider('classification', grading?.provider, null, 'classification check'));
    const resp = await finalProvider.callClassificationApi(output);
    if (!resp.classification) {
        return fail(resp.error || 'Unknown error fetching classification');
    }
    let score;
    if (expected === undefined) {
        score = Math.max(...Object.values(resp.classification));
    }
    else {
        score = resp.classification[expected] || 0;
    }
    if (score >= threshold) {
        const reason = expected === undefined
            ? `Maximum classification score ${score.toFixed(2)} >= ${threshold}`
            : `Classification ${expected} has score ${score.toFixed(2)} >= ${threshold}`;
        return {
            pass: true,
            score,
            reason,
        };
    }
    return {
        pass: false,
        score,
        reason: `Classification ${expected} has score ${score.toFixed(2)} < ${threshold}`,
    };
}
exports.matchesClassification = matchesClassification;
async function matchesLlmRubric(expected, output, grading, vars) {
    if (!grading) {
        throw new Error('Cannot grade output without grading config. Specify --grader option or grading config.');
    }
    const rubricPrompt = grading?.rubricPrompt || prompts_1.DEFAULT_GRADING_PROMPT;
    (0, tiny_invariant_1.default)(typeof rubricPrompt === 'string', 'rubricPrompt must be a string');
    const prompt = nunjucks.renderString(rubricPrompt, {
        output: JSON.stringify(output).slice(1, -1),
        rubric: JSON.stringify(expected).slice(1, -1),
        ...fromVars(vars),
    });
    const finalProvider = await getAndCheckProvider('text', grading.provider, (await (0, defaults_1.getDefaultProviders)()).gradingJsonProvider, 'llm-rubric check');
    const resp = await finalProvider.callApi(prompt);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, tiny_invariant_1.default)(typeof resp.output === 'string', 'llm-rubric produced malformed response');
    try {
        const jsonObjects = (0, util_1.extractJsonObjects)(resp.output);
        if (jsonObjects.length === 0) {
            return fail('Could not extract JSON from llm-rubric response', resp.tokenUsage);
        }
        const parsed = jsonObjects[0];
        const pass = parsed.pass ?? (typeof parsed.score === 'undefined' ? true : parsed.score > 0);
        return {
            pass,
            score: parsed.score ?? (pass ? 1.0 : 0.0),
            reason: parsed.reason || (pass ? 'Grading passed' : 'Grading failed'),
            tokensUsed: {
                total: resp.tokenUsage?.total || 0,
                prompt: resp.tokenUsage?.prompt || 0,
                completion: resp.tokenUsage?.completion || 0,
            },
        };
    }
    catch (err) {
        return fail(`llm-rubric produced malformed response: ${err}\n\n${resp.output}`, resp.tokenUsage);
    }
}
exports.matchesLlmRubric = matchesLlmRubric;
async function matchesFactuality(input, expected, output, grading, vars) {
    if (!grading) {
        throw new Error('Cannot grade output without grading config. Specify --grader option or grading config.');
    }
    const rubricPrompt = grading?.rubricPrompt || prompts_1.OPENAI_FACTUALITY_PROMPT;
    (0, tiny_invariant_1.default)(typeof rubricPrompt === 'string', 'rubricPrompt must be a string');
    const prompt = nunjucks.renderString(rubricPrompt, {
        input: JSON.stringify(input).slice(1, -1),
        ideal: JSON.stringify(expected).slice(1, -1),
        completion: JSON.stringify(output).slice(1, -1),
        ...fromVars(vars),
    });
    const finalProvider = await getAndCheckProvider('text', grading.provider, (await (0, defaults_1.getDefaultProviders)()).gradingProvider, 'factuality check');
    const resp = await finalProvider.callApi(prompt);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, tiny_invariant_1.default)(typeof resp.output === 'string', 'factuality produced malformed response');
    try {
        const output = resp.output;
        // The preferred output starts like "(A)...", but we also support leading whitespace, lowercase letters, and omitting the first parenthesis.
        const answerMatch = output.match(/\s*\(?([a-eA-E])\)/);
        if (!answerMatch) {
            return fail(`Factuality checker output did not match expected format: ${output}`, resp.tokenUsage);
        }
        const option = answerMatch[1].toUpperCase();
        let reason = '';
        const scoreLookup = {
            A: grading.factuality?.subset ?? 1,
            B: grading.factuality?.superset ?? 1,
            C: grading.factuality?.agree ?? 1,
            D: grading.factuality?.disagree ?? 0,
            E: grading.factuality?.differButFactual ?? 1,
        };
        // Passing is defined as scores with value >0, and failing as scores with value 0.
        const passing = Object.keys(scoreLookup).filter((key) => scoreLookup[key] > 0);
        const failing = Object.keys(scoreLookup).filter((key) => scoreLookup[key] === 0);
        let pass = passing.includes(option) && !failing.includes(option);
        const optionReasons = {
            A: `The submitted answer is a subset of the expert answer and is fully consistent with it.`,
            B: `The submitted answer is a superset of the expert answer and is fully consistent with it.`,
            C: `The submitted answer contains all the same details as the expert answer.`,
            D: `There is a disagreement between the submitted answer and the expert answer.`,
            E: `The answers differ, but these differences don't matter from the perspective of factuality.`,
        };
        if (optionReasons[option]) {
            reason = optionReasons[option];
        }
        else {
            pass = false;
            reason = `Invalid option: ${option}. Full response from factuality checker: ${resp.output}`;
        }
        let score = pass ? 1 : 0;
        if (typeof scoreLookup[option] !== 'undefined') {
            score = scoreLookup[option];
        }
        return {
            pass,
            score,
            reason,
            tokensUsed: {
                total: resp.tokenUsage?.total || 0,
                prompt: resp.tokenUsage?.prompt || 0,
                completion: resp.tokenUsage?.completion || 0,
            },
        };
    }
    catch (err) {
        return fail(`Error parsing output: ${err.message}`, resp.tokenUsage);
    }
}
exports.matchesFactuality = matchesFactuality;
async function matchesClosedQa(input, expected, output, grading, vars) {
    if (!grading) {
        throw new Error('Cannot grade output without grading config. Specify --grader option or grading config.');
    }
    const rubricPrompt = grading?.rubricPrompt || prompts_1.OPENAI_CLOSED_QA_PROMPT;
    (0, tiny_invariant_1.default)(typeof rubricPrompt === 'string', 'rubricPrompt must be a string');
    const prompt = nunjucks.renderString(rubricPrompt, {
        input: JSON.stringify(input).slice(1, -1),
        criteria: JSON.stringify(expected).slice(1, -1),
        completion: JSON.stringify(output).slice(1, -1),
        ...fromVars(vars),
    });
    const finalProvider = await getAndCheckProvider('text', grading.provider, (await (0, defaults_1.getDefaultProviders)()).gradingProvider, 'model-graded-closedqa check');
    const resp = await finalProvider.callApi(prompt);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, tiny_invariant_1.default)(typeof resp.output === 'string', 'model-graded-closedqa produced malformed response');
    try {
        const pass = resp.output.endsWith('Y');
        let reason;
        if (pass) {
            reason = 'The submission meets the criterion';
        }
        else if (resp.output.endsWith('N')) {
            reason = `The submission does not meet the criterion:\n${resp.output}`;
        }
        else {
            reason = `Model grader produced a malformed response:\n${resp.output}`;
        }
        return {
            pass,
            score: pass ? 1 : 0,
            reason,
            tokensUsed: {
                total: resp.tokenUsage?.total || 0,
                prompt: resp.tokenUsage?.prompt || 0,
                completion: resp.tokenUsage?.completion || 0,
            },
        };
    }
    catch (err) {
        return fail(`Error parsing output: ${err.message}`, resp.tokenUsage);
    }
}
exports.matchesClosedQa = matchesClosedQa;
async function matchesAnswerRelevance(input, output, threshold, grading) {
    const embeddingProvider = await getAndCheckProvider('embedding', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).embeddingProvider, 'answer relevancy check');
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).gradingProvider, 'answer relevancy check');
    const tokensUsed = {
        total: 0,
        prompt: 0,
        completion: 0,
    };
    const candidateQuestions = [];
    for (let i = 0; i < 3; i++) {
        // TODO(ian): Parallelize
        const rubricPrompt = grading?.rubricPrompt || prompts_1.ANSWER_RELEVANCY_GENERATE;
        (0, tiny_invariant_1.default)(typeof rubricPrompt === 'string', 'rubricPrompt must be a string');
        const promptText = nunjucks.renderString(rubricPrompt, {
            answer: JSON.stringify(output).slice(1, -1),
        });
        const resp = await textProvider.callApi(promptText);
        if (resp.error || !resp.output) {
            tokensUsed.total += resp.tokenUsage?.total || 0;
            tokensUsed.prompt += resp.tokenUsage?.prompt || 0;
            tokensUsed.completion += resp.tokenUsage?.completion || 0;
            return fail(resp.error || 'No output', tokensUsed);
        }
        (0, tiny_invariant_1.default)(typeof resp.output === 'string', 'answer relevancy check produced malformed response');
        candidateQuestions.push(resp.output);
    }
    (0, tiny_invariant_1.default)(typeof embeddingProvider.callEmbeddingApi === 'function', `Provider ${embeddingProvider.id} must implement callEmbeddingApi for similarity check`);
    const inputEmbeddingResp = await embeddingProvider.callEmbeddingApi(input);
    if (inputEmbeddingResp.error || !inputEmbeddingResp.embedding) {
        tokensUsed.total += inputEmbeddingResp.tokenUsage?.total || 0;
        tokensUsed.prompt += inputEmbeddingResp.tokenUsage?.prompt || 0;
        tokensUsed.completion += inputEmbeddingResp.tokenUsage?.completion || 0;
        return fail(inputEmbeddingResp.error || 'No embedding', tokensUsed);
    }
    const inputEmbedding = inputEmbeddingResp.embedding;
    const similarities = [];
    for (const question of candidateQuestions) {
        const resp = await embeddingProvider.callEmbeddingApi(question);
        tokensUsed.total += resp.tokenUsage?.total || 0;
        tokensUsed.prompt += resp.tokenUsage?.prompt || 0;
        tokensUsed.completion += resp.tokenUsage?.completion || 0;
        if (resp.error || !resp.embedding) {
            return fail(resp.error || 'No embedding', tokensUsed);
        }
        similarities.push(cosineSimilarity(inputEmbedding, resp.embedding));
    }
    const similarity = similarities.reduce((a, b) => a + b, 0) / similarities.length;
    const pass = similarity >= threshold;
    const greaterThanReason = `Relevance ${similarity.toFixed(2)} is greater than threshold ${threshold}`;
    const lessThanReason = `Relevance ${similarity.toFixed(2)} is less than threshold ${threshold}`;
    if (pass) {
        return {
            pass: true,
            score: similarity,
            reason: greaterThanReason,
            tokensUsed,
        };
    }
    return {
        pass: false,
        score: similarity,
        reason: lessThanReason,
        tokensUsed,
    };
}
exports.matchesAnswerRelevance = matchesAnswerRelevance;
async function matchesContextRecall(context, groundTruth, threshold, grading, vars) {
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).gradingProvider, 'context recall check');
    const rubricPrompt = grading?.rubricPrompt || prompts_1.CONTEXT_RECALL;
    (0, tiny_invariant_1.default)(typeof rubricPrompt === 'string', 'rubricPrompt must be a string');
    const promptText = nunjucks.renderString(rubricPrompt, {
        context: JSON.stringify(context).slice(1, -1),
        groundTruth: JSON.stringify(groundTruth).slice(1, -1),
        ...fromVars(vars),
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, tiny_invariant_1.default)(typeof resp.output === 'string', 'context-recall produced malformed response');
    const sentences = resp.output.split('\n');
    const numerator = sentences.reduce((acc, sentence) => acc + (sentence.includes(prompts_1.CONTEXT_RECALL_ATTRIBUTED_TOKEN) ? 1 : 0), 0);
    const score = numerator / sentences.length;
    const pass = score >= threshold;
    return {
        pass,
        score,
        reason: pass
            ? `Recall ${score.toFixed(2)} is >= ${threshold}`
            : `Recall ${score.toFixed(2)} is < ${threshold}`,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
        },
    };
}
exports.matchesContextRecall = matchesContextRecall;
async function matchesContextRelevance(question, context, threshold, grading) {
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).gradingProvider, 'context relevance check');
    const rubricPrompt = grading?.rubricPrompt || prompts_1.CONTEXT_RELEVANCE;
    (0, tiny_invariant_1.default)(typeof rubricPrompt === 'string', 'rubricPrompt must be a string');
    const promptText = nunjucks.renderString(rubricPrompt, {
        context: JSON.stringify(context).slice(1, -1),
        query: JSON.stringify(question).slice(1, -1),
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, tiny_invariant_1.default)(typeof resp.output === 'string', 'context-relevance produced malformed response');
    const sentences = resp.output.split('\n');
    const numerator = sentences.reduce((acc, sentence) => acc + (sentence.includes(prompts_1.CONTEXT_RELEVANCE_BAD) ? 0 : 1), 0);
    const score = numerator / sentences.length;
    const pass = score >= threshold;
    return {
        pass,
        score,
        reason: pass
            ? `Relevance ${score.toFixed(2)} is >= ${threshold}`
            : `Relevance ${score.toFixed(2)} is < ${threshold}`,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
        },
    };
}
exports.matchesContextRelevance = matchesContextRelevance;
async function matchesContextFaithfulness(query, output, context, threshold, grading, vars) {
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).gradingProvider, 'faithfulness check');
    if (grading?.rubricPrompt) {
        (0, tiny_invariant_1.default)(Array.isArray(grading.rubricPrompt), 'rubricPrompt must be an array');
    }
    const longformPrompt = grading?.rubricPrompt?.[0] || prompts_1.CONTEXT_FAITHFULNESS_LONGFORM;
    const nliPrompt = grading?.rubricPrompt?.[1] || prompts_1.CONTEXT_FAITHFULNESS_NLI_STATEMENTS;
    let promptText = nunjucks.renderString(longformPrompt, {
        question: JSON.stringify(query).slice(1, -1),
        answer: JSON.stringify(output).slice(1, -1),
        ...fromVars(vars),
    });
    let resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, tiny_invariant_1.default)(typeof resp.output === 'string', 'context-faithfulness produced malformed response');
    const statements = resp.output.split('\n');
    promptText = nunjucks.renderString(nliPrompt, {
        context: JSON.stringify(context).slice(1, -1),
        statements: JSON.stringify(statements.join('\n')).slice(1, -1),
        ...fromVars(vars),
    });
    resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return fail(resp.error || 'No output', resp.tokenUsage);
    }
    (0, tiny_invariant_1.default)(typeof resp.output === 'string', 'context-faithfulness produced malformed response');
    let finalAnswer = 'Final verdict for each statement in order:';
    finalAnswer = finalAnswer.toLowerCase();
    let verdicts = resp.output.toLowerCase().trim();
    let score;
    if (verdicts.includes(finalAnswer)) {
        verdicts = verdicts.slice(verdicts.indexOf(finalAnswer) + finalAnswer.length);
        score =
            verdicts.split('.').filter((answer) => answer.trim() !== '' && !answer.includes('yes'))
                .length / statements.length;
    }
    else {
        score = (verdicts.split('verdict: no').length - 1) / statements.length;
    }
    score = 1 - score;
    const pass = score >= threshold;
    return {
        pass,
        score,
        reason: pass
            ? `Faithfulness ${score.toFixed(2)} is >= ${threshold}`
            : `Faithfulness ${score.toFixed(2)} is < ${threshold}`,
        tokensUsed: {
            total: resp.tokenUsage?.total || 0,
            prompt: resp.tokenUsage?.prompt || 0,
            completion: resp.tokenUsage?.completion || 0,
        },
    };
}
exports.matchesContextFaithfulness = matchesContextFaithfulness;
async function matchesSelectBest(criteria, outputs, grading, vars) {
    (0, tiny_invariant_1.default)(outputs.length >= 2, 'select-best assertion must have at least two outputs to compare between');
    const textProvider = await getAndCheckProvider('text', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).gradingProvider, 'select-best check');
    const rubricPrompt = grading?.rubricPrompt || prompts_1.SELECT_BEST_PROMPT;
    (0, tiny_invariant_1.default)(typeof rubricPrompt === 'string', 'rubricPrompt must be a string');
    const promptText = nunjucks.renderString(rubricPrompt, {
        criteria: JSON.stringify(criteria).slice(1, -1),
        outputs: outputs.map((output) => JSON.stringify(output).slice(1, -1)),
        ...fromVars(vars),
    });
    const resp = await textProvider.callApi(promptText);
    if (resp.error || !resp.output) {
        return new Array(outputs.length).fill(fail(resp.error || 'No output', resp.tokenUsage));
    }
    (0, tiny_invariant_1.default)(typeof resp.output === 'string', 'select-best produced malformed response');
    const firstDigitMatch = resp.output.trim().match(/\d/);
    const verdict = firstDigitMatch ? parseInt(firstDigitMatch[0], 10) : NaN;
    if (isNaN(verdict) || verdict < 0 || verdict >= outputs.length) {
        return new Array(outputs.length).fill(fail(`Invalid select-best verdict: ${verdict}`));
    }
    const tokensUsed = {
        total: resp.tokenUsage?.total || 0,
        prompt: resp.tokenUsage?.prompt || 0,
        completion: resp.tokenUsage?.completion || 0,
    };
    return outputs.map((output, index) => {
        if (index === verdict) {
            return {
                pass: true,
                score: 1,
                reason: `Output selected as the best: ${criteria}`,
                tokensUsed,
            };
        }
        else {
            return {
                pass: false,
                score: 0,
                reason: `Output not selected: ${criteria}`,
                tokensUsed,
            };
        }
    });
}
exports.matchesSelectBest = matchesSelectBest;
async function matchesModeration({ userPrompt, assistantResponse, categories = [] }, grading) {
    const moderationProvider = (await getAndCheckProvider('moderation', grading?.provider, (await (0, defaults_1.getDefaultProviders)()).moderationProvider, 'moderation check'));
    (0, tiny_invariant_1.default)(moderationProvider, 'Moderation provider must be defined');
    const resp = await moderationProvider.callModerationApi(userPrompt, assistantResponse);
    if (resp.error) {
        return {
            pass: false,
            score: 0,
            reason: `Moderation API error: ${resp.error}`,
        };
    }
    const { flags } = resp;
    if (!flags || flags.length === 0) {
        return {
            pass: true,
            score: 1,
            reason: 'No moderation flags detected',
        };
    }
    const filteredFlags = categories.length === 0 ? flags : flags.filter((flag) => categories.includes(flag.code));
    if (filteredFlags.length > 0) {
        return {
            pass: false,
            score: 0,
            reason: `Moderation flags detected: ${filteredFlags
                .map((flag) => flag.description)
                .join(', ')}`,
        };
    }
    return {
        pass: true,
        score: 1,
        reason: 'No relevant moderation flags detected',
    };
}
exports.matchesModeration = matchesModeration;
//# sourceMappingURL=matchers.js.map