"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
/**
 * LangSmith Evaluation Script for summarizeFormWithLLM
 *
 * Evaluates the   function by:
 * - Running test cases from LangSmith dataset
 * - Checking if a summary is provided
 * - Generating evaluation reports
 *
 * Requirements:
 * - Set environment variables in .env:
 *   - OPENAI_API_KEY
 *   - LANGSMITH_API_KEY
 *   - LANGSMITH_TRACING=true
 *   - LANGSMITH_PROJECT=ai-actions-local
 *
 * Usage:
 * yarn ts-node extensions/shelly/lib/summarizeFormWithLLM/summarizeFormWithLLM.evaluate.ts
 *
 * Results can be viewed in LangSmith dashboard
 *
 * ⚠️ **Note:** This script does NOT run in CI/CD. It is meant for **manual evaluation** before merging PRs (for now)
 */
const langsmith_1 = require("langsmith");
const evaluation_1 = require("langsmith/evaluation");
const createOpenAIModel_1 = require("../../../../src/lib/llm/openai/createOpenAIModel");
const summarizeFormWithLLM_1 = require("./summarizeFormWithLLM");
const constants_1 = require("../../../../src/lib/llm/openai/constants");
const lodash_1 = require("lodash");
const dotenv_1 = __importDefault(require("dotenv"));
dotenv_1.default.config();
const langsmith = new langsmith_1.Client({
    apiKey: process.env.LANGSMITH_API_KEY,
    apiUrl: process.env.LANGSMITH_ENDPOINT,
});
// Define the dataset name
const datasetName = 'ai-actions-summarize-form-shelly';
// Fetch the 'test' split examples from the dataset
const fetchTestExamples = async () => {
    try {
        const testExamples = langsmith.listExamples({
            datasetName,
            splits: ['test'],
        });
        const examples = [];
        for await (const example of testExamples) {
            if (!(0, lodash_1.isNil)(example.inputs)) {
                // Explicit null check
                examples.push(example);
            }
        }
        return examples;
    }
    catch (error) {
        console.error('❌ Error fetching test examples:', error);
        throw error;
    }
};
// Custom evaluator to check if a summary is provided
// Actual evaluation will be run automatically in LangSmith by LLM as a judge
// TODO: right now we are usign gpt4o as a judge as well - not problematic but not great either - same models tend to prefer each other styles
const summaryProvidedEvaluator = async ({ outputs, }) => {
    const summary = outputs === null || outputs === void 0 ? void 0 : outputs.summary;
    // Check if summary is provided and not empty
    const isProvided = typeof summary === 'string' && summary.trim().length > 0;
    return {
        key: 'summary_provided',
        score: isProvided ? 1 : 0,
        metadata: {
            summaryLength: typeof summary === 'string' ? summary.length : 0,
            hasSummary: isProvided
        }
    };
};
// Wrapper function to adapt summarizeFormWithLLM for evaluation
const summarizeFormWithLLMWrapper = async (input) => {
    const payload = {
        activity: {
            id: 'test-activity-id',
        },
        pathway: {
            tenant_id: 'test-tenant',
            definition_id: 'test-definition',
            id: 'test-pathway',
            org_slug: 'test-org',
            org_id: 'test-org-id',
        },
        fields: {
            form_data: input.form_data,
            format: input.format,
            language: input.language,
            disclaimer_message: input.disclaimer_message,
        },
        settings: {
            openAiApiKey: process.env.OPENAI_API_KEY,
        },
    };
    const helpers = {
        getOpenAIConfig: () => {
            const apiKey = process.env.OPENAI_API_KEY;
            if ((0, lodash_1.isNil)(apiKey) || apiKey.trim() === '') {
                throw new Error('OPENAI_API_KEY is required but not set');
            }
            return { apiKey };
        },
    };
    const { model, metadata, callbacks } = await (0, createOpenAIModel_1.createOpenAIModel)({
        settings: payload.settings,
        helpers,
        payload,
        modelType: constants_1.OPENAI_MODELS.GPT4o,
    });
    const summary = await (0, summarizeFormWithLLM_1.summarizeFormWithLLM)({
        model,
        formData: input.form_data,
        summaryFormat: input.format,
        language: input.language,
        disclaimerMessage: input.disclaimer_message,
        metadata,
        callbacks,
    });
    return { summary };
};
// Main function to run the evaluation and print results
const runEvaluation = async () => {
    var _a, _b;
    try {
        console.log('📡 Fetching test dataset from LangSmith...');
        const testExamples = await fetchTestExamples();
        console.log(`✅ Loaded ${testExamples.length} test examples\n`);
        console.log('🚀 Running evaluation...\n');
        const results = await (0, evaluation_1.evaluate)(summarizeFormWithLLMWrapper, {
            data: testExamples,
            evaluators: [summaryProvidedEvaluator],
            experimentPrefix: 'SummarizeFormWithLLM Evaluation',
            maxConcurrency: 4,
            numRepetitions: 5,
        });
        const resultsArray = Array.isArray(results) ? results : [results];
        const experimentId = (_b = (_a = resultsArray[0]) === null || _a === void 0 ? void 0 : _a.run) === null || _b === void 0 ? void 0 : _b.run_id;
        console.log('\n✨ Experiment Run Complete! Actual evaluation is running in LangSmith via LLM as judges.');
        console.log('View detailed results in LangSmith:');
        console.log(`https://smith.langchain.com/runs/${experimentId}`);
    }
    catch (error) {
        console.error('❌ Error during evaluation:', error);
        throw error;
    }
};
// Execute the evaluation
void runEvaluation();
//# sourceMappingURL=summarizeFormWithLLM.evaluate.js.map