/*
 * Copyright OpenSearch Contributors
 * SPDX-License-Identifier: Apache-2.0
 */
/**
 * Evaluation Service
 * Main orchestrator for running agent evaluations
 */
import { v4 as uuidv4 } from 'uuid';
import { executeBeforeRequestHook, executeAfterResponseHook } from '../../lib/hooks.js';
import { AGUIToTrajectoryConverter, consumeSSEStream, buildAgentPayload } from '../../services/agent/index.js';
import { generateMockTrajectory } from './mockTrajectory.js';
import { callBedrockJudge } from './bedrockJudge.js';
import { buildJudgeMatcherEntry, formatExpectedOutcomesAsClaim } from '../../lib/matchers/judgeAccessor.js';
import { buildJudgeAgentsHints } from '../../services/traces/judgeAgentsHints.js';
// Re-export for use by experimentRunner when calling judge after trace polling
export { callBedrockJudge };
import { openSearchClient } from '../../services/opensearch/index.js';
import { debug } from '../../lib/debug.js';
import { ENV_CONFIG } from '../../lib/config.js';
import { DEFAULT_CONFIG } from '../../lib/constants.js';
// For browser/Vite builds, use DEFAULT_CONFIG directly.
// For server/CLI (Node.js), we can use loadConfigSync from lib/config/index.
// This is a runtime check to avoid importing Node.js modules in browser builds.
const getModels = () => {
    // Check if we're in a Node.js environment with file system access
    if (typeof window === 'undefined' && typeof process !== 'undefined' && process.versions?.node) {
        try {
            // Dynamic import to avoid bundling issues
            // eslint-disable-next-line @typescript-eslint/no-require-imports
            const { loadConfigSync } = require('../../lib/config/index.js');
            return loadConfigSync().models;
        }
        catch {
            // Fall back to defaults
        }
    }
    return DEFAULT_CONFIG.models;
};
// Toggle between mock and real agent
const USE_MOCK_AGENT = false;
/**
 * Build ConnectorAuth from AgentConfig.
 * Prefers explicit `auth` field if present, falls back to header inference.
 */
function buildConnectorAuth(agent) {
    // Prefer explicit auth config (new pattern)
    if (agent.auth && agent.auth.type !== 'none') {
        return {
            ...agent.auth,
            // Merge any extra headers on top
            headers: { ...agent.headers, ...agent.auth.headers },
        };
    }
    // Legacy: infer from headers
    const headers = agent.headers || {};
    if (headers['Authorization']?.startsWith('Bearer ')) {
        return {
            type: 'bearer',
            token: headers['Authorization'].replace('Bearer ', ''),
        };
    }
    if (headers['Authorization']?.startsWith('Basic ')) {
        return {
            type: 'basic',
            token: headers['Authorization'].replace('Basic ', ''),
        };
    }
    if (headers['x-api-key']) {
        return {
            type: 'api-key',
            token: headers['x-api-key'],
        };
    }
    // Pass through all headers
    return {
        type: 'none',
        headers,
    };
}
/**
 * Drive a single agent invocation through its connector and return the raw
 * trajectory/runId/rawEvents — no judge, no report synthesis.
 *
 * Owns the connector resolution, request building, auth, the
 * `beforeRequest`/`afterResponse` hooks, and execution timing. Extracted from
 * {@link runEvaluationWithConnector} so the RFC-004 engine can offer an
 * `agent.run()` fixture that captures exactly the same trajectory/trace
 * correlation the legacy path produces (see RFC 004 §4.1, #256).
 */
export async function invokeAgent(agent, modelId, testCase, options) {
    const { registry: connectorRegistry, onStep, onRawEvent, env: runEnv } = options;
    // Get connector for this agent
    const agentWithConnector = agent;
    const connector = connectorRegistry.getForAgent(agentWithConnector);
    // Merge any per-invocation env (from the SDK's AgentRunOptions.env) into the
    // connector config so subprocess connectors forward it to the spawned child.
    // Static config env stays the base; per-call env wins on key collisions.
    const baseConnectorConfig = agentWithConnector.connectorConfig;
    const mergedConnectorConfig = runEnv && Object.keys(runEnv).length > 0
        ? {
            ...(baseConnectorConfig || {}),
            env: { ...(baseConnectorConfig?.env || {}), ...runEnv },
        }
        : baseConnectorConfig;
    // Build connector request
    let request = {
        testCase,
        modelId,
        connectorConfig: mergedConnectorConfig,
    };
    // Build auth from agent config
    const auth = buildConnectorAuth(agent);
    // Execute beforeRequest hook if defined
    let effectiveEndpoint = agent.endpoint;
    if (agent.hooks?.beforeRequest) {
        const previewPayload = connector.buildPayload(request);
        const hookContext = {
            endpoint: agent.endpoint,
            payload: previewPayload,
            headers: auth.headers || agent.headers || {},
        };
        const hookResult = await executeBeforeRequestHook(agent.hooks, hookContext, agent.key);
        effectiveEndpoint = hookResult.endpoint;
        // Pass the hook-modified payload through to the connector so it skips
        // its internal buildPayload() call. This preserves ALL modifications the
        // hook made to the payload (threadId, runId, custom fields, etc.)
        request = {
            ...request,
            payload: hookResult.payload,
        };
        // Merge any hook-modified headers into auth
        if (hookResult.headers) {
            auth.headers = { ...auth.headers, ...hookResult.headers };
        }
    }
    // Execute via connector (with timing)
    const agentStartTime = Date.now();
    let result = await connector.execute(effectiveEndpoint, request, auth, onStep, onRawEvent);
    const agentDurationMs = Date.now() - agentStartTime;
    // Execute afterResponse hook if defined
    if (agent.hooks?.afterResponse) {
        debug('Eval', `Executing afterResponse hook for agent "${agent.key}"`);
        try {
            const hookContext = {
                response: (result.rawEvents?.length ? result.rawEvents[result.rawEvents.length - 1] : null) || result.metadata || {},
                trajectory: result.trajectory,
                runId: result.runId || undefined,
                rawEvents: result.rawEvents || [],
                metadata: result.metadata,
            };
            const hookResult = await executeAfterResponseHook(agent.hooks, hookContext, agent.key);
            // Apply hook modifications
            result = {
                ...result,
                trajectory: hookResult.trajectory,
                runId: hookResult.runId || result.runId,
            };
            debug('Eval', 'afterResponse hook applied:', {
                trajectorySteps: hookResult.trajectory.length,
                runId: hookResult.runId
            });
        }
        catch (hookError) {
            const errorMsg = hookError instanceof Error ? hookError.message : String(hookError);
            console.error(`[Eval] afterResponse hook failed for agent "${agent.key}":`, errorMsg);
            debug('Eval', `afterResponse hook error details:`, {
                agent: agent.key,
                error: errorMsg,
                rawEventsCount: result.rawEvents?.length ?? 0,
                hasMetadata: !!result.metadata,
            });
            // Re-throw so the caller knows the hook failed — don't silently swallow
            throw hookError instanceof Error ? hookError : new Error(errorMsg);
        }
    }
    return {
        trajectory: result.trajectory,
        runId: result.runId,
        rawEvents: result.rawEvents || [],
        agentDurationMs,
        connector,
    };
}
export async function runEvaluationWithConnector(agent, modelId, testCase, onStep, options) {
    const { registry: connectorRegistry, onRawEvent, evaluatorId, skipJudge } = options;
    // Pull `judgeModelId` once so the closure inside the standard-mode call
    // below uses the run-level value rather than the agent's `modelId`.
    const optionsJudgeModelId = options.judgeModelId;
    const reportId = uuidv4();
    let fullTrajectory = [];
    let rawEvents = [];
    let agentRunId = null;
    debug('Eval', 'Config:', { agent: agent.name, model: modelId, testCase: testCase.id });
    const evalStartTime = Date.now();
    try {
        // Drive the agent through its connector (connector resolution, request
        // building, auth, before/afterResponse hooks, timing) via the shared
        // primitive. Judge + report synthesis stay here in the legacy path.
        const invocation = await invokeAgent(agent, modelId, testCase, {
            registry: connectorRegistry,
            onStep,
            onRawEvent,
        });
        const connector = invocation.connector;
        const agentDurationMs = invocation.agentDurationMs;
        fullTrajectory = invocation.trajectory;
        agentRunId = invocation.runId;
        rawEvents = invocation.rawEvents;
        debug('Eval', 'Trajectory captured:', fullTrajectory.length, 'steps');
        debug('Eval', 'Raw events captured:', rawEvents.length);
        // TRACE MODE: Skip logs fetch and judge, return pending report
        if (agent.useTraces) {
            return {
                id: reportId,
                timestamp: new Date().toISOString(),
                agentName: agent.name,
                agentKey: agent.key,
                modelName: modelId,
                modelId: modelId,
                testCaseId: testCase.id,
                testCaseVersion: testCase.currentVersion ?? 1,
                status: 'completed',
                metricsStatus: 'pending',
                trajectory: fullTrajectory,
                metrics: {
                    accuracy: 0,
                    faithfulness: 0,
                    latency_score: 0,
                    trajectory_alignment_score: 0,
                },
                llmJudgeReasoning: 'Waiting for traces to become available...',
                improvementStrategies: [],
                runId: agentRunId || undefined,
                rawEvents,
                connectorProtocol: connector.type,
                performanceMetrics: {
                    durationMs: Date.now() - evalStartTime,
                    agentDurationMs,
                },
            };
        }
        // SKIP JUDGE MODE: Return report without judge evaluation (caller handles it)
        if (skipJudge) {
            return {
                id: reportId,
                timestamp: new Date().toISOString(),
                agentName: agent.name,
                agentKey: agent.key,
                modelName: modelId,
                modelId,
                testCaseId: testCase.id,
                testCaseVersion: testCase.currentVersion ?? 1,
                status: 'completed',
                trajectory: fullTrajectory,
                metrics: { accuracy: 0, faithfulness: 0, latency_score: 0, trajectory_alignment_score: 0 },
                llmJudgeReasoning: '',
                improvementStrategies: [],
                runId: agentRunId || undefined,
                rawEvents,
                connectorProtocol: connector.type,
                performanceMetrics: {
                    durationMs: Date.now() - evalStartTime,
                    agentDurationMs,
                },
            };
        }
        // STANDARD MODE: Call judge
        const models = getModels();
        const modelConfig = models[modelId];
        // Judge model resolution — SEPARATE from the agent's `modelId`. Priority:
        //   1. options.judgeModelId  (run-level customer input)
        //   2. server default `BEDROCK_MODEL_ID`  (env, falls back to a recent Claude)
        //   3. agent's modelId  (last-resort BC fallback for old callers that didn't
        //                       split agent vs judge model; tolerated for one release
        //                       so behavior of pre-existing benchmark runs is
        //                       preserved when neither cx input nor server default
        //                       is available)
        // Anything else (evaluator's own `inferenceConfig.modelId`, agentic-provider
        // model picking) is resolved server-side in /api/judge from the evaluator.
        const judgeModelId = optionsJudgeModelId ||
            process.env.BEDROCK_MODEL_ID ||
            modelConfig?.model_id ||
            modelId;
        const judgment = await callBedrockJudge(fullTrajectory, {
            expectedOutcomes: testCase.expectedOutcomes,
            expectedTrajectory: testCase.expectedTrajectory,
        }, undefined, // No logs in direct connector mode
        (chunk) => debug('Eval', 'Judge progress:', chunk.slice(0, 100)), judgeModelId, evaluatorId, 
        // Forward agent runId so the `agent` (trace) judge provider can
        // scope its query_spans/query_logs tools. See callBedrockJudge.
        agentRunId || undefined, 
        // Strategy C correlation hints (#264) so the trace judge tool can
        // find spans the agent emits under its OWN correlation (claude-code
        // session ids etc.), not just spans matching agent-health's runId
        // via gen_ai.request.id.
        buildJudgeAgentsHints({
            agentKey: agent.key,
            connectorProtocol: agent.connectorType,
            timestamp: new Date().toISOString(),
            performanceMetrics: { durationMs: Date.now() - evalStartTime, agentDurationMs },
        }, agent.traceServiceName));
        debug('Eval', 'Metrics:', judgment.metrics);
        const llmJudgeResponse = {
            modelId: judgeModelId,
            timestamp: new Date().toISOString(),
            promptTokens: 0,
            completionTokens: 0,
            latencyMs: judgment.judgeDurationMs ?? 0,
            // Prefer the actual unparsed model output when present (post
            // evaluator-prompt-plumbing). Pre-fix code stuffed the parsed
            // reasoning here — fall back to that for back-compat with judges
            // that don't yet forward the raw text.
            rawResponse: judgment.rawResponse ?? judgment.llmJudgeReasoning,
            parsedMetrics: judgment.metrics,
            improvementStrategies: judgment.improvementStrategies,
            ...(judgment.extraFields ? { extraFields: judgment.extraFields } : {}),
            ...(judgment.judgeDebug ? { judgeDebug: judgment.judgeDebug } : {}),
        };
        return {
            id: reportId,
            timestamp: new Date().toISOString(),
            agentName: agent.name,
            agentKey: agent.key,
            modelName: modelId,
            modelId,
            testCaseId: testCase.id,
            testCaseVersion: testCase.currentVersion ?? 1,
            status: 'completed',
            passFailStatus: judgment.passFailStatus,
            trajectory: fullTrajectory,
            metrics: judgment.metrics,
            llmJudgeReasoning: judgment.llmJudgeReasoning,
            // Unified judge surface (Option-B BC: legacy field above kept).
            matcherResults: [
                buildJudgeMatcherEntry(judgment, {
                    claim: formatExpectedOutcomesAsClaim(testCase.expectedOutcomes),
                    model: judgeModelId,
                }),
            ],
            improvementStrategies: judgment.improvementStrategies,
            llmJudgeResponse,
            runId: agentRunId || undefined,
            rawEvents,
            connectorProtocol: connector.type,
            performanceMetrics: {
                durationMs: Date.now() - evalStartTime,
                agentDurationMs,
                judgeDurationMs: judgment.judgeDurationMs,
                judgeAttempts: judgment.judgeAttempts,
            },
        };
    }
    catch (error) {
        console.error('[Eval] Error:', error instanceof Error ? error.message : error);
        // Enhanced debug logging for connection failures
        if (error instanceof Error) {
            debug('Eval', 'Error details:', {
                name: error.name,
                message: error.message,
                stack: error.stack,
                cause: error.cause,
                agent: agent.name,
                endpoint: agent.endpoint,
                modelId,
                testCaseId: testCase.id,
            });
        }
        else {
            debug('Eval', 'Unknown error:', error);
        }
        // Get connector type for error case (may not be available if error was in getting connector)
        let connectorType;
        try {
            const agentWithConnector = agent;
            const connector = connectorRegistry.getForAgent(agentWithConnector);
            connectorType = connector.type;
            debug('Eval', 'Connector type:', connectorType);
        }
        catch (connectorError) {
            debug('Eval', 'Failed to get connector type:', connectorError);
            // Connector lookup failed, leave undefined
        }
        return {
            id: reportId,
            timestamp: new Date().toISOString(),
            agentName: agent.name,
            agentKey: agent.key,
            modelName: modelId,
            modelId,
            testCaseId: testCase.id,
            testCaseVersion: testCase.currentVersion ?? 1,
            status: 'failed',
            trajectory: fullTrajectory,
            metrics: {
                accuracy: 0,
                faithfulness: 0,
                latency_score: 0,
                trajectory_alignment_score: 0,
            },
            llmJudgeReasoning: `Evaluation failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
            improvementStrategies: [],
            rawEvents,
            connectorProtocol: connectorType,
        };
    }
}
/**
 * Run real agent evaluation by streaming AG UI events
 * @deprecated Use runEvaluationWithConnector() via the server's /api/evaluate endpoint instead.
 * This browser-side path is kept for backwards compatibility but has no active callers.
 */
async function runRealAgentEvaluation(agent, modelId, testCase, onStep, onRawEvent) {
    const trajectory = [];
    const rawEvents = [];
    const converter = new AGUIToTrajectoryConverter();
    const agentPayload = buildAgentPayload(testCase, modelId);
    debug('Eval', 'Agent payload:', JSON.stringify(agentPayload).substring(0, 500));
    // Use proxy to avoid CORS issues when calling agent endpoint
    const proxyPayload = {
        endpoint: agent.endpoint,
        payload: agentPayload,
        headers: agent.headers,
        agentKey: agent.key,
    };
    debug('Eval', 'Using proxy:', ENV_CONFIG.agentProxyUrl);
    await consumeSSEStream(ENV_CONFIG.agentProxyUrl, proxyPayload, (event) => {
        // Capture raw event for debugging
        rawEvents.push(event);
        onRawEvent?.(event);
        // Convert to trajectory steps
        const steps = converter.processEvent(event);
        steps.forEach(step => {
            trajectory.push(step);
            onStep(step);
        });
    });
    const runId = converter.getRunId();
    return { trajectory, runId, rawEvents };
}
/**
 * Run evaluation with selected agent, model, and test case
 * Streams trajectory steps to UI in real-time via onStep callback
 *
 * @deprecated Use runServerEvaluation() from services/client/evaluationApi instead.
 * The server-side path (/api/evaluate) consolidates all evaluation logic through the
 * connector system, ensuring consistent behavior for hooks, storage, and all agent types.
 */
export async function runEvaluation(agent, modelId, testCase, onStep, onRawEvent) {
    const reportId = uuidv4();
    let fullTrajectory = [];
    let rawEvents = [];
    let agentRunId = null;
    debug('Eval', 'Config:', { agent: agent.name, model: modelId, testCase: testCase.id });
    const evalStartTime = Date.now();
    try {
        if (USE_MOCK_AGENT) {
            fullTrajectory = await generateMockTrajectory(testCase);
            for (const step of fullTrajectory) {
                onStep(step);
                await new Promise(r => setTimeout(r, 300));
            }
        }
        else {
            const result = await runRealAgentEvaluation(agent, modelId, testCase, onStep, onRawEvent);
            fullTrajectory = result.trajectory;
            agentRunId = result.runId;
            rawEvents = result.rawEvents;
        }
        debug('Eval', 'Trajectory captured:', fullTrajectory.length, 'steps');
        debug('Eval', 'Raw events captured:', rawEvents.length);
        // TRACE MODE: Skip logs fetch and judge, return pending report
        // Traces take ~5 minutes to propagate, so we'll poll for them later
        if (agent.useTraces) {
            return {
                id: reportId,
                timestamp: new Date().toISOString(),
                agentName: agent.name,
                agentKey: agent.key,
                modelName: modelId,
                modelId: modelId,
                testCaseId: testCase.id,
                testCaseVersion: testCase.currentVersion ?? 1,
                status: 'completed',
                metricsStatus: 'pending', // Will be updated after traces are available
                trajectory: fullTrajectory,
                metrics: {
                    accuracy: 0,
                    faithfulness: 0,
                    latency_score: 0,
                    trajectory_alignment_score: 0,
                },
                llmJudgeReasoning: 'Waiting for traces to become available...',
                improvementStrategies: [],
                runId: agentRunId || undefined,
                rawEvents,
            };
        }
        // STANDARD MODE: Fetch logs and call judge immediately
        let logs;
        if (agentRunId) {
            try {
                logs = await openSearchClient.fetchLogsForRun(agentRunId);
            }
            catch (error) {
                console.error('[Eval] Failed to fetch logs:', error instanceof Error ? error.message : error);
            }
        }
        // Call judge
        const models = getModels();
        const modelConfig = models[modelId];
        // Deprecated path — mirror the standard-mode judgeModelId resolution
        // chain so this stays consistent if anyone still hits this code path.
        // The agent's `modelId` is NOT used as the judge model here either.
        const judgeModelId = process.env.BEDROCK_MODEL_ID ||
            modelConfig?.model_id ||
            modelId;
        const judgment = await callBedrockJudge(fullTrajectory, {
            expectedOutcomes: testCase.expectedOutcomes,
            expectedTrajectory: testCase.expectedTrajectory,
        }, logs, (chunk) => debug('Eval', 'Judge progress:', chunk.slice(0, 100)), judgeModelId, undefined, // evaluatorId not threaded on the legacy path
        // Same forwarding as the standard branch above.
        agentRunId || undefined);
        debug('Eval', 'Metrics:', judgment.metrics);
        const llmJudgeResponse = {
            modelId: judgeModelId,
            timestamp: new Date().toISOString(),
            promptTokens: 0,
            completionTokens: 0,
            latencyMs: judgment.judgeDurationMs ?? 0,
            // Prefer the actual unparsed model output when present (post
            // evaluator-prompt-plumbing). Pre-fix code stuffed the parsed
            // reasoning here — fall back to that for back-compat.
            rawResponse: judgment.rawResponse ?? judgment.llmJudgeReasoning,
            parsedMetrics: judgment.metrics,
            improvementStrategies: judgment.improvementStrategies,
            ...(judgment.extraFields ? { extraFields: judgment.extraFields } : {}),
            ...(judgment.judgeDebug ? { judgeDebug: judgment.judgeDebug } : {}),
        };
        return {
            id: reportId,
            timestamp: new Date().toISOString(),
            agentName: agent.name,
            agentKey: agent.key,
            modelName: modelId,
            modelId,
            testCaseId: testCase.id,
            testCaseVersion: testCase.currentVersion ?? 1,
            status: 'completed',
            passFailStatus: judgment.passFailStatus,
            trajectory: fullTrajectory,
            metrics: judgment.metrics,
            llmJudgeReasoning: judgment.llmJudgeReasoning,
            // Unified judge surface (Option-B BC: legacy field above kept).
            matcherResults: [
                buildJudgeMatcherEntry(judgment, {
                    claim: formatExpectedOutcomesAsClaim(testCase.expectedOutcomes),
                    model: judgeModelId,
                }),
            ],
            improvementStrategies: judgment.improvementStrategies,
            llmJudgeResponse,
            openSearchLogs: logs,
            runId: agentRunId || undefined,
            logs: logs || undefined,
            rawEvents,
        };
    }
    catch (error) {
        console.error('[Eval] Error:', error instanceof Error ? error.message : error);
        return {
            id: reportId,
            timestamp: new Date().toISOString(),
            agentName: agent.name,
            agentKey: agent.key,
            modelName: modelId,
            modelId,
            testCaseId: testCase.id,
            testCaseVersion: testCase.currentVersion ?? 1,
            status: 'failed',
            trajectory: fullTrajectory,
            metrics: {
                accuracy: 0,
                faithfulness: 0,
                latency_score: 0,
                trajectory_alignment_score: 0,
            },
            llmJudgeReasoning: `Evaluation failed: ${error instanceof Error ? error.message : 'Unknown error'}`,
            improvementStrategies: [],
            rawEvents,
        };
    }
}
//# sourceMappingURL=index.js.map