import { env } from "../../../env.js";
import manageDbPool_default from "../../../data/db/manageDbPool.js";
import runDbClient_default from "../../../data/db/runDbClient.js";
import { getLogger as getLogger$1 } from "../../../logger.js";
import manageDbClient_default from "../../../data/db/manageDbClient.js";
import { GENERATION_TYPES, ModelFactory, getConversationHistory, getFullAgent, getProjectScopedRef, resolveRef, withRef } from "@inkeep/agents-core";
import { context, propagation } from "@opentelemetry/api";
import { Output, generateText } from "ai";
import { z } from "zod";

//#region src/domains/evals/services/EvaluationService.ts
const logger = getLogger$1("EvaluationService");
function withOtelBaggage(entries, fn) {
	const bag = Object.entries(entries).reduce((b, [key, value]) => b.setEntry(key, { value }), propagation.getBaggage(context.active()) ?? propagation.createBaggage());
	return context.with(propagation.setBaggage(context.active(), bag), fn);
}
var EvaluationService = class {
	manageApiBypassSecret;
	constructor() {
		this.manageApiBypassSecret = env.INKEEP_AGENTS_MANAGE_API_BYPASS_SECRET;
	}
	/**
	* Execute an evaluation by calling the LLM with the evaluator prompt and conversation data
	*/
	async executeEvaluation(params) {
		const { conversation, evaluator, tenantId, projectId, expectedOutput } = params;
		let resolvedRef = null;
		if (conversation.ref) resolvedRef = conversation.ref;
		else {
			const ref = getProjectScopedRef(tenantId, projectId, "main");
			resolvedRef = await resolveRef(manageDbClient_default)(ref);
		}
		if (!resolvedRef) throw new Error("Failed to resolve ref");
		const conversationHistory = await getConversationHistory(runDbClient_default)({
			scopes: {
				tenantId,
				projectId
			},
			conversationId: conversation.id,
			options: {
				includeInternal: false,
				limit: 100
			}
		});
		let agentDefinition = null;
		let agentId = null;
		try {
			agentId = conversation.agentId ?? null;
			if (agentId) {
				const agentIdForLookup = agentId;
				agentDefinition = await withRef(manageDbPool_default, resolvedRef, (db) => getFullAgent(db)({ scopes: {
					tenantId,
					projectId,
					agentId: agentIdForLookup
				} }));
			} else logger.warn({
				conversationId: conversation.id,
				agentId: conversation.agentId
			}, "AgentId not found, cannot get agent definition");
		} catch (error) {
			logger.warn({
				error,
				conversationId: conversation.id,
				agentId: conversation.agentId
			}, "Failed to fetch agent definition for evaluation");
		}
		const prettifiedTrace = await this.fetchTraceFromSigNoz({
			conversationId: conversation.id,
			tenantId,
			projectId
		});
		logger.info({
			conversationId: conversation.id,
			hasTrace: !!prettifiedTrace,
			traceActivityCount: prettifiedTrace?.timeline?.length || 0
		}, "Trace fetch completed");
		const conversationText = JSON.stringify(conversationHistory, null, 2);
		const agentDefinitionText = agentDefinition ? JSON.stringify(agentDefinition, null, 2) : "Agent definition not available";
		const traceText = prettifiedTrace ? JSON.stringify(prettifiedTrace, null, 2) : "Trace data not available";
		const modelConfig = evaluator.model ?? {};
		let schemaObj;
		if (typeof evaluator.schema === "string") try {
			schemaObj = JSON.parse(evaluator.schema);
		} catch (error) {
			logger.error({
				error,
				schemaString: evaluator.schema
			}, "Failed to parse evaluator schema string");
			throw new Error("Invalid evaluator schema format");
		}
		else schemaObj = evaluator.schema;
		logger.info({
			evaluatorId: evaluator.id,
			schemaType: typeof schemaObj,
			schemaKeys: schemaObj && typeof schemaObj === "object" ? Object.keys(schemaObj) : []
		}, "Using evaluator schema");
		const expectedOutputText = expectedOutput ? JSON.stringify(expectedOutput, null, 2) : void 0;
		const evaluationPrompt = this.buildEvalInputEvaluationPrompt(evaluator.prompt, agentDefinitionText, conversationText, traceText, schemaObj, expectedOutputText);
		const llmResponse = await this.callLLM({
			prompt: evaluationPrompt,
			modelConfig,
			schema: schemaObj,
			tenantId,
			projectId,
			conversationId: conversation.id,
			agentId
		});
		return {
			output: llmResponse.result,
			metadata: {
				...llmResponse.metadata,
				model: modelConfig.model || "unknown",
				agentId,
				hasAgentDefinition: !!agentDefinition,
				hasTrace: !!prettifiedTrace,
				traceActivityCount: prettifiedTrace?.timeline?.length || 0
			}
		};
	}
	/**
	* Build evaluation prompt with agent definition, conversation history, trace, and expected output
	*/
	buildEvalInputEvaluationPrompt(evaluatorPrompt, agentDefinitionText, conversationText, traceText, schema, expectedOutputText) {
		const schemaDescription = JSON.stringify(schema, null, 2);
		return `${evaluatorPrompt}

Agent Definition:

${agentDefinitionText}

Conversation History:

${conversationText}

Execution Trace:

${traceText}
${expectedOutputText ? `

Expected Output:

${expectedOutputText}
` : ""}
Please evaluate this conversation according to the following schema and return your evaluation as JSON:

${schemaDescription}

Return your evaluation as a JSON object matching the schema above.`;
	}
	/**
	* Call LLM API using AI SDK's generateText with structured output
	*/
	async callLLM(params) {
		const { prompt, modelConfig, schema, tenantId, projectId, conversationId, agentId } = params;
		const languageModel = ModelFactory.prepareGenerationConfig(modelConfig);
		const providerOptions = modelConfig?.providerOptions || {};
		let resultSchema;
		try {
			resultSchema = z.fromJSONSchema(schema);
			logger.info({
				schemaType: typeof schema,
				schemaKeys: schema && typeof schema === "object" ? Object.keys(schema) : [],
				convertedSchema: "success"
			}, "Converted JSON schema to Zod");
		} catch (error) {
			logger.error({
				error,
				schema
			}, "Failed to convert JSON schema to Zod, using fallback");
			resultSchema = z.record(z.string(), z.unknown());
		}
		const evaluationSchema = resultSchema;
		try {
			logger.info({
				promptLength: prompt.length,
				model: modelConfig.model
			}, "Calling generateText with structured output for eval scoring");
			const result = await withOtelBaggage({
				"tenant.id": tenantId,
				"project.id": projectId,
				"conversation.id": conversationId,
				...agentId ? { "agent.id": agentId } : {}
			}, () => generateText({
				...languageModel,
				output: Output.object({ schema: evaluationSchema }),
				prompt,
				temperature: providerOptions.temperature ?? .3,
				experimental_telemetry: {
					isEnabled: true,
					metadata: {
						tenantId,
						projectId,
						conversationId,
						agentId: agentId ?? "",
						generationType: GENERATION_TYPES.EVAL_SCORING
					}
				}
			}));
			return {
				result: result.output,
				metadata: { usage: result.usage }
			};
		} catch (error) {
			const errorMessage = error instanceof Error ? error.message : String(error);
			logger.error({
				error: errorMessage,
				schema: JSON.stringify(schema, null, 2),
				promptPreview: prompt.substring(0, 500)
			}, "Evaluation failed with generateText structured output");
			throw new Error(`Evaluation failed: ${errorMessage}`);
		}
	}
	async fetchTraceFromSigNoz(params) {
		const { conversationId, tenantId, projectId } = params;
		const manageUIUrl = env.INKEEP_AGENTS_MANAGE_UI_URL;
		const maxRetries = 2;
		const retryDelayMs = 2e4;
		const initialDelayMs = 3e4;
		const traceUrl = `${manageUIUrl}/api/traces/conversations/${conversationId}?tenantId=${tenantId}&projectId=${projectId}`;
		try {
			logger.info({
				conversationId,
				manageUIUrl,
				initialDelayMs
			}, "Waiting 30s before fetching trace from SigNoz");
			await new Promise((resolve) => setTimeout(resolve, initialDelayMs));
			for (let attempt = 0; attempt <= maxRetries; attempt++) try {
				logger.info({
					conversationId,
					attempt: attempt + 1,
					maxRetries: maxRetries + 1
				}, "Fetching trace from SigNoz");
				const headers$1 = {};
				if (this.manageApiBypassSecret) headers$1.Authorization = `Bearer ${this.manageApiBypassSecret}`;
				const traceResponse = await fetch(traceUrl, { headers: headers$1 });
				if (!traceResponse.ok) {
					logger.warn({
						conversationId,
						status: traceResponse.status,
						statusText: traceResponse.statusText,
						attempt: attempt + 1
					}, "Failed to fetch trace from SigNoz");
					if (attempt < maxRetries) {
						logger.info({
							conversationId,
							retryDelayMs
						}, "Retrying trace fetch after delay");
						await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
						continue;
					}
					return null;
				}
				const conversationDetail = await traceResponse.json();
				logger.debug({
					conversationId,
					activityTypes: conversationDetail.activities?.map((a) => a.type) || [],
					activityCount: conversationDetail.activities?.length || 0
				}, "Checking activities for ai_assistant_message type");
				if (!conversationDetail.activities?.some((activity) => activity.type === "ai_assistant_message")) {
					logger.warn({
						conversationId,
						attempt: attempt + 1,
						activityCount: conversationDetail.activities?.length || 0,
						activityTypes: conversationDetail.activities?.slice(0, 5).map((a) => a.type) || []
					}, "Trace fetched but ai_assistant_message not found in activities");
					if (attempt < maxRetries) {
						logger.info({
							conversationId,
							retryDelayMs
						}, "Retrying trace fetch after delay to wait for assistant message");
						await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
						continue;
					}
					logger.warn({
						conversationId,
						maxRetries,
						activityCount: conversationDetail.activities?.length || 0
					}, "Max retries reached, ai_assistant_message not found - proceeding with available trace data");
				} else logger.info({
					conversationId,
					activityCount: conversationDetail.activities?.length || 0,
					attempt: attempt + 1
				}, "Trace fetched successfully with ai_assistant_message");
				return this.formatConversationAsPrettifiedTrace(conversationDetail);
			} catch (fetchError) {
				logger.warn({
					error: fetchError,
					conversationId,
					attempt: attempt + 1
				}, "Error fetching trace from SigNoz");
				if (attempt < maxRetries) {
					logger.info({
						conversationId,
						retryDelayMs
					}, "Retrying trace fetch after delay");
					await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
					continue;
				}
				return null;
			}
			return null;
		} catch (error) {
			logger.warn({
				error,
				conversationId,
				manageUIUrl
			}, "Failed to fetch trace from SigNoz, will continue without trace");
			return null;
		}
	}
	/**
	* Format conversation detail as prettified trace
	*/
	formatConversationAsPrettifiedTrace(conversation) {
		return {
			metadata: {
				conversationId: conversation.conversationId,
				traceId: conversation.traceId,
				agentName: conversation.agentName,
				agentId: conversation.agentId,
				exportedAt: (/* @__PURE__ */ new Date()).toISOString()
			},
			timing: {
				startTime: conversation.conversationStartTime || "",
				endTime: conversation.conversationEndTime || "",
				durationMs: conversation.duration || 0
			},
			timeline: (conversation.activities || []).map((activity) => {
				const { id: _id, ...rest } = activity;
				return { ...rest };
			})
		};
	}
};

//#endregion
export { EvaluationService };