"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
/**
 * LangSmith Evaluation Script for detectLanguageWithLLM
 *
 * Evaluates the function by:
 * - Running test cases from LangSmith dataset
 * - Checking if a language is detected
 * - Generating evaluation reports
 *
 * Requirements:
 * - Set environment variables in .env:
 *   - OPENAI_API_KEY
 *   - LANGSMITH_API_KEY
 *   - LANGSMITH_TRACING=true
 *   - LANGSMITH_PROJECT=ai-actions-local
 *
 * Usage:
 * yarn ts-node extensions/shelly/lib/detectLanguageWithLLM/detectLanguageWithLLM.evaluate.ts
 *
 * Results can be viewed in LangSmith dashboard
 *
 * ⚠️ **Note:** This script does NOT run in CI/CD. It is meant for **manual evaluation** before merging PRs (for now)
 */
const langsmith_1 = require("langsmith");
const evaluation_1 = require("langsmith/evaluation");
const createOpenAIModel_1 = require("../../../../src/lib/llm/openai/createOpenAIModel");
const detectLanguageWithLLM_1 = require("./detectLanguageWithLLM");
const constants_1 = require("../../../../src/lib/llm/openai/constants");
const lodash_1 = require("lodash");
const dotenv_1 = __importDefault(require("dotenv"));
dotenv_1.default.config();
const langsmith = new langsmith_1.Client({
    apiKey: process.env.LANGSMITH_API_KEY,
    apiUrl: process.env.LANGSMITH_ENDPOINT,
});
// Define the dataset name
const datasetName = 'ai-actions-detect-language-shelly';
// Fetch the 'test' split examples from the dataset
const fetchTestExamples = async () => {
    try {
        const testExamples = langsmith.listExamples({
            datasetName,
            splits: ['test'],
        });
        const examples = [];
        for await (const example of testExamples) {
            if (!(0, lodash_1.isNil)(example.inputs)) {
                // Explicit null check
                examples.push(example);
            }
        }
        return examples;
    }
    catch (error) {
        console.error('❌ Error fetching test examples:', error);
        throw error;
    }
};
// Custom evaluator to check if language detection is accurate
const languageDetectionEvaluator = async ({ outputs, }) => {
    const detectedLanguage = outputs === null || outputs === void 0 ? void 0 : outputs.detectedLanguage;
    // Check if language is detected and not empty
    const isDetected = typeof detectedLanguage === 'string' && detectedLanguage.trim().length > 0;
    return {
        key: 'language_detected',
        score: isDetected ? 1 : 0,
        metadata: {
            detectedLanguage,
            isDetected
        }
    };
};
// Wrapper function to adapt detectLanguageWithLLM for evaluation
const detectLanguageWithLLMWrapper = async (input) => {
    const payload = {
        activity: {
            id: 'test-activity-id',
        },
        pathway: {
            tenant_id: 'test-tenant',
            definition_id: 'test-definition',
            id: 'test-pathway',
            org_slug: 'test-org',
            org_id: 'test-org-id',
        },
        fields: {
            text: input.text,
        },
        settings: {
            openAiApiKey: process.env.OPENAI_API_KEY,
        },
    };
    const helpers = {
        getOpenAIConfig: () => {
            const apiKey = process.env.OPENAI_API_KEY;
            if ((0, lodash_1.isNil)(apiKey) || apiKey.trim() === '') {
                throw new Error('OPENAI_API_KEY is required but not set');
            }
            return { apiKey };
        },
    };
    const { model, metadata, callbacks } = await (0, createOpenAIModel_1.createOpenAIModel)({
        settings: payload.settings,
        helpers,
        payload,
        modelType: constants_1.OPENAI_MODELS.GPT4o,
    });
    const detectedLanguage = await (0, detectLanguageWithLLM_1.detectLanguageWithLLM)({
        model,
        text: input.text,
        metadata,
        callbacks,
    });
    return { detectedLanguage };
};
// Main function to run the evaluation and print results
const runEvaluation = async () => {
    var _a, _b;
    try {
        console.log('📡 Fetching test dataset from LangSmith...');
        const testExamples = await fetchTestExamples();
        console.log(`✅ Loaded ${testExamples.length} test examples\n`);
        console.log('🚀 Running evaluation...\n');
        const results = await (0, evaluation_1.evaluate)(detectLanguageWithLLMWrapper, {
            data: testExamples,
            evaluators: [languageDetectionEvaluator],
            experimentPrefix: 'DetectLanguageWithLLM Evaluation',
            maxConcurrency: 8,
            numRepetitions: 1,
        });
        const resultsArray = Array.isArray(results) ? results : [results];
        const experimentId = (_b = (_a = resultsArray[0]) === null || _a === void 0 ? void 0 : _a.run) === null || _b === void 0 ? void 0 : _b.run_id;
        console.log('\n✨ Experiment Run Complete! Actual evaluation is running in LangSmith via custom evaluator.');
        console.log('View detailed results in LangSmith:');
        console.log(`https://smith.langchain.com/runs/${experimentId}`);
    }
    catch (error) {
        console.error('❌ Error during evaluation:', error);
        throw error;
    }
};
// Execute the evaluation
void runEvaluation();
//# sourceMappingURL=detectLanguageWithLLM.evaluate.js.map