"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
/**
 * LangSmith Evaluation Script for findAppointmentsWithLLM
 *
 * Evaluates the findAppointmentsWithLLM function by:
 * - Running test cases from LangSmith dataset
 * - Comparing generated appointments against expected appointments
 * - Generating evaluation reports
 *
 * Requirements:
 * - Set environment variables in .env:
 *   - OPENAI_API_KEY
 *   - LANGSMITH_API_KEY
 *   - LANGSMITH_TRACING=true
 *   - LANGSMITH_PROJECT=ai-actions-local
 *
 * Usage:
 * yarn ts-node extensions/elation/lib/findAppointmentsWithLLM/findAppointmentsWithLLM.evaluate.ts
 *
 * Results can be viewed in LangSmith dashboard:
 * https://smith.langchain.com/o/3fffae83-70ff-4574-81ba-aaaedf0b4dc5/datasets/745cea13-3379-463f-9a8a-c6b10e29b8f6
 *
 *  * ⚠️ **Note:** This script does NOT run in CI/CD. It is meant for **manual evaluation** before merging PRs (for now)
 */
const langsmith_1 = require("langsmith");
const evaluation_1 = require("langsmith/evaluation");
const createOpenAIModel_1 = require("../../../../src/lib/llm/openai/createOpenAIModel");
const findAppointmentsWithLLM_1 = require("./findAppointmentsWithLLM");
const constants_1 = require("../../../../src/lib/llm/openai/constants");
const lodash_1 = require("lodash");
const dotenv_1 = __importDefault(require("dotenv"));
dotenv_1.default.config();
const langsmith = new langsmith_1.Client({
    apiKey: process.env.LANGSMITH_API_KEY,
    apiUrl: process.env.LANGSMITH_ENDPOINT,
});
// Define the dataset name
const datasetName = 'ai-actions-find-appointments-elation';
// Fetch the 'test' split examples from the dataset
const fetchTestExamples = async () => {
    try {
        const testExamples = langsmith.listExamples({
            datasetName,
            splits: ['test'], // change to ['validation'] to run on validation set
        });
        const examples = [];
        for await (const example of testExamples) {
            if (!(0, lodash_1.isNil)(example.inputs) && !(0, lodash_1.isNil)(example.outputs)) {
                // Explicit null check
                examples.push(example);
            }
        }
        return examples;
    }
    catch (error) {
        console.error('❌ Error fetching test examples:', error);
        throw error;
    }
};
// Custom evaluator to compare generated appointments with expected appointments
const appointmentsMatchEvaluator = async ({ outputs, referenceOutputs, }) => {
    const generatedAppointmentIds = outputs === null || outputs === void 0 ? void 0 : outputs.appointmentIds;
    const expectedAppointmentIds = referenceOutputs === null || referenceOutputs === void 0 ? void 0 : referenceOutputs.expected_output_appointment_ids;
    const isEqual = Array.isArray(generatedAppointmentIds) &&
        Array.isArray(expectedAppointmentIds) &&
        generatedAppointmentIds.length === expectedAppointmentIds.length &&
        generatedAppointmentIds.every((id, index) => id === expectedAppointmentIds[index]);
    return {
        key: 'appointments_match',
        score: isEqual ? 1 : 0,
        metadata: {
            generatedIds: generatedAppointmentIds,
            expectedIds: expectedAppointmentIds
        }
    };
};
// Wrapper function to adapt findAppointmentsWithLLM for evaluation
const findAppointmentsWithLLMWrapper = async (input) => {
    const payload = {
        activity: {
            id: 'test-activity-id',
        },
        pathway: {
            tenant_id: 'test-tenant',
            definition_id: 'test-definition',
            id: 'test-pathway',
            org_slug: 'test-org',
            org_id: 'test-org-id',
        },
        fields: {
            instructions: input.instruction,
            patientId: 'test-patient',
        },
        settings: {
            openAiApiKey: process.env.OPENAI_API_KEY,
        },
    };
    const helpers = {
        getOpenAIConfig: () => {
            const apiKey = process.env.OPENAI_API_KEY;
            if ((0, lodash_1.isNil)(apiKey) || apiKey.trim() === '') {
                throw new Error('OPENAI_API_KEY is required but not set');
            }
            return { apiKey };
        },
    };
    const { model, metadata, callbacks } = await (0, createOpenAIModel_1.createOpenAIModel)({
        settings: payload.settings,
        helpers,
        payload,
        modelType: constants_1.OPENAI_MODELS.GPT4o,
    });
    return await (0, findAppointmentsWithLLM_1.findAppointmentsWithLLM)({
        model,
        appointments: input.input_appointments,
        prompt: input.instruction,
        metadata,
        callbacks,
    });
};
// Main function to run the evaluation and print results
const runEvaluation = async () => {
    var _a, _b;
    try {
        console.log('📡 Fetching test dataset from LangSmith...');
        const testExamples = await fetchTestExamples();
        console.log(`✅ Loaded ${testExamples.length} test examples\n`);
        console.log('🚀 Running evaluation...\n');
        const results = await (0, evaluation_1.evaluate)(findAppointmentsWithLLMWrapper, {
            data: testExamples,
            evaluators: [appointmentsMatchEvaluator],
            experimentPrefix: 'FindAppointmentsWithLLM Evaluation',
            maxConcurrency: 16,
            numRepetitions: 3,
        });
        const resultsArray = Array.isArray(results) ? results : [results];
        const experimentId = (_b = (_a = resultsArray[0]) === null || _a === void 0 ? void 0 : _a.run) === null || _b === void 0 ? void 0 : _b.run_id;
        console.log('\n✨ Evaluation Complete!');
        console.log('View detailed results in LangSmith:');
        console.log(`https://smith.langchain.com/runs/${experimentId}`);
    }
    catch (error) {
        console.error('❌ Error during evaluation:', error);
        throw error;
    }
};
// Execute the evaluation
void runEvaluation();
//# sourceMappingURL=FindAppointmentsWithLLM.evaluate.js.map