"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
/**
 * LangSmith Evaluation Script for checkPatientTags
 *
 * Evaluates the checkPatientTags function by:
 * - Running test cases from LangSmith dataset
 * - Comparing generated tag checks against expected results
 * - Generating evaluation reports
 *
 * Requirements:
 * - Set environment variables in .env:
 *   - OPENAI_API_KEY
 *   - LANGSMITH_API_KEY
 *   - LANGSMITH_TRACING=true
 *   - LANGSMITH_PROJECT=ai-actions-local
 *
 * Usage:
 *  * yarn ts-node extensions/elation/actions/checkPatientTags/lib/evaluateCheckTags.ts
 *
 * Results can be viewed in LangSmith dashboard:
 * https://smith.langchain.com/o/3fffae83-70ff-4574-81ba-aaaedf0b4dc5/datasets/2b3df5fd-6266-43f2-a701-6e5d9981eacc/compare?selectedSessions=2bcc6e49-4087-4051-99f1-6a5d6fce7ac2
 *
 *  * ⚠️ **Note:** This script does NOT run in CI/CD. It is meant for **manual evaluation** before merging PRs (for now)
 */
const langsmith_1 = require("langsmith");
const evaluation_1 = require("langsmith/evaluation");
const createOpenAIModel_1 = require("../../../../../src/lib/llm/openai/createOpenAIModel");
const checkTagsWithLLM_1 = require("./checkTagsWithLLM/checkTagsWithLLM");
const constants_1 = require("../../../../../src/lib/llm/openai/constants");
const lodash_1 = require("lodash");
const dotenv_1 = __importDefault(require("dotenv"));
dotenv_1.default.config();
const langsmith = new langsmith_1.Client({
    apiKey: process.env.LANGSMITH_API_KEY,
    apiUrl: process.env.LANGSMITH_ENDPOINT,
});
// Define the dataset name
const datasetName = 'ai-actions-check-patient-tags-elation';
// Fetch the 'test' split examples from the dataset
const fetchTestExamples = async () => {
    try {
        const testExamples = langsmith.listExamples({
            datasetName,
            splits: ['test'],
        });
        const examples = [];
        for await (const example of testExamples) {
            if (!(0, lodash_1.isNil)(example.inputs) && !(0, lodash_1.isNil)(example.outputs)) {
                // Explicit null check
                examples.push(example);
            }
        }
        return examples;
    }
    catch (error) {
        console.error('❌ Error fetching test examples:', error);
        throw error;
    }
};
// Custom evaluator to compare generated tag check results with expected results
const tagCheckEvaluator = async ({ outputs, referenceOutputs, }) => {
    console.log('Evaluator received:', { outputs, referenceOutputs }); // Debug log
    const generatedResult = outputs === null || outputs === void 0 ? void 0 : outputs.tagsFound;
    const expectedResult = referenceOutputs === null || referenceOutputs === void 0 ? void 0 : referenceOutputs.expected_output;
    // console.log('Comparing results:', { generatedResult, expectedResult }); // Debug log
    return {
        key: 'tag_check_match',
        score: generatedResult === expectedResult ? 1 : 0
    };
};
// Wrapper function to adapt checkTagsWithLLM for evaluation
const checkTagsWithLLMWrapper = async (input) => {
    const payload = {
        activity: {
            id: 'test-activity-id',
        },
        pathway: {
            tenant_id: 'test-tenant',
            definition_id: 'test-definition',
            id: 'test-pathway',
            org_slug: 'test-org',
            org_id: 'test-org-id',
        },
        fields: {
            instructions: input.instruction,
            patientId: 'test-patient',
        },
        settings: {
            openAiApiKey: process.env.OPENAI_API_KEY,
        },
    };
    const helpers = {
        getOpenAIConfig: () => {
            const apiKey = process.env.OPENAI_API_KEY;
            if ((0, lodash_1.isNil)(apiKey) || apiKey.trim() === '') {
                throw new Error('OPENAI_API_KEY is required but not set');
            }
            return { apiKey };
        },
    };
    const { model, metadata, callbacks } = await (0, createOpenAIModel_1.createOpenAIModel)({
        settings: payload.settings,
        helpers,
        payload,
        modelType: constants_1.OPENAI_MODELS.GPT4o,
    });
    return await (0, checkTagsWithLLM_1.checkTagsWithLLM)({
        model,
        existingTags: input.input_patient_tags,
        instructions: input.instruction,
        metadata,
        callbacks,
    });
};
// Main function to run the evaluation and print results
const runEvaluation = async () => {
    var _a, _b;
    try {
        console.log('📡 Fetching test dataset from LangSmith...');
        const testExamples = await fetchTestExamples();
        console.log(`✅ Loaded ${testExamples.length} test examples\n`);
        console.log('🚀 Running evaluation...\n');
        const results = await (0, evaluation_1.evaluate)(checkTagsWithLLMWrapper, {
            data: testExamples,
            evaluators: [tagCheckEvaluator],
            experimentPrefix: 'CheckTagsWithLLM Evaluation',
            maxConcurrency: 16,
        });
        const resultsArray = Array.isArray(results) ? results : [results];
        const experimentId = (_b = (_a = resultsArray[0]) === null || _a === void 0 ? void 0 : _a.run) === null || _b === void 0 ? void 0 : _b.run_id;
        if (!(0, lodash_1.isNil)(experimentId) && experimentId.trim() !== '') {
            console.log('\n✨ Evaluation Complete!');
            console.log('View detailed results in LangSmith:');
            console.log(`https://smith.langchain.com/runs/${experimentId}`);
        }
    }
    catch (error) {
        console.error('❌ Error during evaluation:', error);
        throw error;
    }
};
// Execute the evaluation
void runEvaluation();
//# sourceMappingURL=evaluateCheckTags.js.map