"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.handleMultimodalToolCorrectness = void 0;
// import { spawn } from "child_process";
// import path from "path";
// import type { AssertionParams, GradingResult } from "../types";
// export const handleMultimodalToolCorrectness = async ({
//   assertion,
//   outputString,
//   test,
// }: AssertionParams): Promise<GradingResult> => {
//   const scriptPath = path.join(__dirname, "python", "multimodal_tool_correctness_eval.py");
//   const context = {
//     vars: {
//       image_path: test.vars?.image_path,
//       query: test.vars?.query
//     }
//   };
//   const jsonPromise = new Promise<string>((resolve, reject) => {
//     const py = spawn("python", [scriptPath, outputString, JSON.stringify(context)]);
//     let out = "", err = "";
//     py.stdout.on("data", d => (out += d.toString()));
//     py.stderr.on("data", d => (err += d.toString()));
//     py.on("close", code => code === 0 ? resolve(out.trim()) : reject(err));
//     py.on("error", reject);
//   });
//   try {
//     const result = JSON.parse(await jsonPromise);
//     return {
//       assertion,
//       pass: result.pass,
//       score: result.score,
//       reason: result.reason,
//     };
//   } catch (e) {
//     return { assertion, pass: false, score: 0, reason: `Failed: ${e}` };
//   }
// };
const child_process_1 = require("child_process");
const path_1 = __importDefault(require("path"));
const matchers_1 = require("../matchers");
const defaults_1 = require("../providers/defaults");
const invariant_1 = __importDefault(require("../util/invariant"));
async function executeAndGradeMultimodalToolCorrectness(llmOutput, imageProvider, context) {
    const scriptPath = path_1.default.join(__dirname, 'python', 'multimodal_tool_correctness_eval.py');
    const contextJson = JSON.stringify(context);
    const resultPromise = new Promise((resolve, reject) => {
        const py = (0, child_process_1.spawn)('python', [scriptPath, llmOutput, contextJson], {
            env: process.env,
            stdio: ['pipe', 'pipe', 'pipe'],
        });
        let stdout = '', stderr = '';
        py.stdin.write(JSON.stringify(imageProvider));
        py.stdin.end();
        py.stdout.on('data', (d) => (stdout += d.toString()));
        py.stderr.on('data', (d) => {
            stderr += d.toString();
            console.error('Python STDERR:', d.toString());
        });
        py.on('close', (code) => {
            console.error('Python STDERR(final):', stderr.trim());
            if (code === 0) {
                resolve(stdout.trim());
            }
            else {
                reject(new Error(`Python exited ${code}. stdout="${stdout}", stderr="${stderr}"`));
            }
        });
        py.on('error', (e) => reject(new Error(`Failed to start Python: ${e.message}`)));
    });
    try {
        return JSON.parse(await resultPromise);
    }
    catch (e) {
        console.error('Invalid JSON from Python:', e);
        return { pass: false, score: 0, reason: 'evaluation failed' };
    }
}
const handleMultimodalToolCorrectness = async ({ assertion, outputString, test, workspaceId, }) => {
    (0, invariant_1.default)(typeof outputString === 'string', 'output must be string');
    const imageProvider = await (0, matchers_1.getAndCheckProvider)('text', test.options?.provider, (await (0, defaults_1.getDefaultProviders)(undefined, workspaceId)).gradingProvider, 'multimodal-toolCorrectness');
    const modelName = imageProvider.modelName;
    console.log('MultimodalToolCorrectness provider config:', imageProvider.config);
    console.log('MultimodalToolCorrectness modelName:', modelName);
    const contextForPython = {
        ...test,
        defaultTest: test.defaultTest?.assert || [],
        options: {
            provider: { modelName },
        },
    };
    console.log('Context for Python:', contextForPython);
    const grading = await executeAndGradeMultimodalToolCorrectness(outputString, imageProvider, contextForPython);
    return {
        assertion,
        pass: grading.pass,
        score: grading.score,
        reason: grading.reason,
    };
};
exports.handleMultimodalToolCorrectness = handleMultimodalToolCorrectness;
//# sourceMappingURL=handleMultimodalToolCorrectness.js.map