"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const openai_1 = require("@langchain/openai");
const langsmith_1 = require("langsmith");
const evaluation_1 = require("langsmith/evaluation");
const openai_2 = require("../../../../src/lib/llm/openai");
const parser_1 = require("./parser");
const prompt_1 = require("./prompt");
const date_fns_1 = require("date-fns");
const dotenv_1 = __importDefault(require("dotenv"));
dotenv_1.default.config();
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const langsmith = new langsmith_1.Client({
    apiKey: process.env.LANGSMITH_API_KEY,
    apiUrl: process.env.LANGSMITH_ENDPOINT,
});
// Define the application logic you want to evaluate inside a target function
// The SDK will automatically send the inputs from the dataset to your target function
async function wrapper(inputs) {
    const model = new openai_1.ChatOpenAI({
        modelName: openai_2.OPENAI_MODELS.GPT4o,
        openAIApiKey: process.env.OPENAI_API_KEY,
    });
    const chain = model.pipe(parser_1.parser);
    const result = await chain.invoke(await prompt_1.systemPrompt.format({
        currentDateTime: inputs.reference_date,
        prompt: inputs.instructions,
    }));
    return {
        from: result.from,
        to: result.to,
    };
}
const evaluator = async ({ outputs, referenceOutputs, }) => {
    const isCorrect = (0, date_fns_1.differenceInMinutes)((0, date_fns_1.parseISO)(outputs.from), (0, date_fns_1.parseISO)(referenceOutputs === null || referenceOutputs === void 0 ? void 0 : referenceOutputs.from)) < 1 &&
        (0, date_fns_1.differenceInMinutes)((0, date_fns_1.parseISO)(outputs.to), (0, date_fns_1.parseISO)(referenceOutputs === null || referenceOutputs === void 0 ? void 0 : referenceOutputs.to)) <
            1;
    return {
        key: 'extract_dates_from_instructions',
        score: isCorrect ? 1 : 0,
    };
};
async function runEvaluation() {
    var _a, _b;
    try {
        console.log('🚀 Running evaluation...\n');
        const results = await (0, evaluation_1.evaluate)(wrapper, {
            data: 'ai-actions-extract-date-instructions',
            evaluators: [evaluator],
            experimentPrefix: 'ExtractDatesFromInstructions',
            maxConcurrency: 16,
        });
        const resultsArray = Array.isArray(results) ? results : [results];
        const experimentId = (_b = (_a = resultsArray[0]) === null || _a === void 0 ? void 0 : _a.run) === null || _b === void 0 ? void 0 : _b.run_id;
        console.log('\n✨ Evaluation Complete!');
        console.log('View detailed results in LangSmith:');
        console.log(`https://smith.langchain.com/runs/${experimentId}`);
    }
    catch (error) {
        console.error('❌ Error during evaluation:', error);
        throw error;
    }
}
void runEvaluation();
//# sourceMappingURL=evaluate.js.map