"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.handleMultimodalFaithfulness = void 0;
const child_process_1 = require("child_process");
const path_1 = __importDefault(require("path"));
const matchers_1 = require("../matchers");
const defaults_1 = require("../providers/defaults");
const invariant_1 = __importDefault(require("../util/invariant"));
async function executeAndGradeMultimodalFaithfulness(outputMarkdown, imageProvider, context) {
    const pythonScriptPath = path_1.default.join(__dirname, 'python', 'multimodal_faithfulness_eval.py');
    const contextJson = JSON.stringify(context);
    const jsonPromise = new Promise((resolve, reject) => {
        const py = (0, child_process_1.spawn)('python', [pythonScriptPath, outputMarkdown, contextJson], {
            env: process.env,
            stdio: ['pipe', 'pipe', 'pipe'],
        });
        let stdoutData = '', stderrData = '';
        py.stdin.write(JSON.stringify(imageProvider));
        py.stdin.end();
        py.stdout.on('data', (c) => (stdoutData += c.toString()));
        py.stderr.on('data', (c) => {
            stderrData += c.toString();
            console.error('Python STDERR:', c.toString());
        });
        py.on('close', (code) => {
            console.error('Python STDERR(final):', stderrData.trim());
            if (code === 0) {
                resolve(stdoutData.trim());
            }
            else {
                reject(new Error(`Python exited ${code}. stdout="${stdoutData}", stderr="${stderrData}"`));
            }
        });
        py.on('error', (err) => reject(new Error(`Failed to start Python: ${err.message}`)));
    });
    try {
        return JSON.parse(await jsonPromise);
    }
    catch (e) {
        console.error('Invalid JSON from Python:', e);
        return { pass: false, score: 0, reason: 'evaluation failed' };
    }
}
const handleMultimodalFaithfulness = async ({ assertion, outputString, test, workspaceId, }) => {
    (0, invariant_1.default)(typeof outputString === 'string', 'output must be string');
    const imageProvider = await (0, matchers_1.getAndCheckProvider)('text', test.options?.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).gradingProvider, 'multimodal-faithfulness');
    console.log('MultimodalFaithfulness provider config:', imageProvider.config);
    const modelName = imageProvider.modelName;
    console.log('MultimodalFaithfulness modelName:', modelName);
    // ensure image context is properly wrapped for the metric
    const wrappedOutput = outputString.replace(/(!\[.*?\]\(.*?\))/g, 'Before this statement.\n$1\nAfter this statement.');
    const contextForPython = {
        ...test,
        defaultTest: test.defaultTest?.assert || [],
        options: { provider: { modelName } },
        retrieval_context: test.vars?.retrieval_context || [],
    };
    console.log('Context for Python:', contextForPython);
    const grading = await executeAndGradeMultimodalFaithfulness(wrappedOutput, imageProvider, contextForPython);
    return {
        assertion,
        pass: grading.pass,
        score: grading.score,
        reason: grading.reason,
    };
};
exports.handleMultimodalFaithfulness = handleMultimodalFaithfulness;
//# sourceMappingURL=handleMultimodalFaithfulness.js.map