/**
 * Judge executor for evaluating AI-generated applications.
 *
 * Orchestrates the evaluation process using LLM with browser
 * automation capabilities through Playwright MCP.
 */
import { safeJsonStringify } from "../../util/log-sanitizer.js";
import { PlaywrightBridge } from "./integration/playwright-bridge.js";
import { buildJudgeSystemPrompt } from "./prompts/system-prompt.js";
import { buildJudgeTools } from "./tools/index.js";
/**
 * Default judge configuration.
 */
const DEFAULT_JUDGE_CONFIG = {
    maxSteps: 30,
    timeoutMs: 5 * 60 * 1000, // 5 minutes
    passingThreshold: 70,
    captureScreenshots: true,
    detailedReasoning: true,
};
/**
 * Executes judge evaluations for AI-generated applications.
 *
 * Uses LLM with Playwright browser automation to comprehensively
 * evaluate applications against defined criteria.
 */
export class JudgeExecutor {
    llmClient;
    model;
    logger;
    playwrightBridge;
    config;
    constructor(llmClient, model, logger, config) {
        this.llmClient = llmClient;
        this.model = model;
        this.logger = logger;
        this.config = {
            ...DEFAULT_JUDGE_CONFIG,
            ...config,
        };
        this.playwrightBridge = new PlaywrightBridge(this.config.playwrightMcpUrl, logger);
    }
    /**
     * Initializes the judge executor.
     *
     * Sets up connections to required services.
     */
    async initialize() {
        await this.playwrightBridge.initialize();
        this.logger.info("Judge executor initialized");
    }
    /**
     * Evaluates a simulation result against criteria.
     *
     * @param simulationResult - The simulation to evaluate
     * @param criteria - Evaluation criteria
     * @param appUrl - URL of the application to evaluate
     * @returns Comprehensive evaluation result
     */
    async evaluateSimulation(simulationResult, criteria, appUrl) {
        const startTime = Date.now();
        this.logger.info(`Starting evaluation for ${simulationResult.promptId} at ${appUrl}. Criteria: ${criteria.functionalRequirements.length} functional, ${criteria.uiRequirements.length} UI, ${criteria.dataRequirements.length} data, ${criteria.performanceRequirements?.length || 0} performance`);
        try {
            // Build judge context for tools
            const _judgeContext = {
                appUrl,
                prompts: simulationResult.stepResults.map((sr) => sr.prompt),
                criteria,
                evaluationState: {
                    criteriaChecked: [],
                    evidence: new Map(),
                    scores: new Map(),
                },
            };
            // Build tools with Playwright bridge
            this.logger.info("Building tools");
            const tools = (await buildJudgeTools(this.playwrightBridge));
            // Build system prompt
            this.logger.info("Building system prompt");
            const systemPrompt = buildJudgeSystemPrompt(criteria, this.config);
            // Build user prompt
            this.logger.info("Building user prompt");
            const userPrompt = this.buildUserPrompt(simulationResult, criteria, appUrl);
            // Track evaluation result
            let evaluation = null;
            let tokenUsage = { input: 0, output: 0, cached: 0, total: 0 };
            // Execute judge with llmClient
            this.logger.info("Starting evaluation with config:", safeJsonStringify(this.config));
            const maxSteps = this.config.maxSteps || 5;
            const result = await this.llmClient.streamText({
                model: this.model,
                system: systemPrompt,
                user: {
                    role: "user",
                    content: userPrompt,
                },
                context: {
                    contextId: {
                        appId: "judge",
                        userId: "system",
                        name: `judge-${simulationResult.promptId}-${Date.now()}`,
                    },
                    contextOptionsV2: {
                        contextLimit: 100_000,
                    },
                    // Note: Judge contexts are ephemeral (not persisted)
                },
                tools,
                maxSteps,
                providerConfig: {
                    provider: "vertex",
                    model: "claude-sonnet-4-5",
                },
                prepareStep: (step) => {
                    const remainingSteps = maxSteps - step.stepNumber - 1;
                    const halfwayPoint = Math.floor(maxSteps / 2);
                    this.logger.info(`Preparing step ${step.stepNumber} (maxSteps=${maxSteps}, remaining=${remainingSteps})`);
                    // On the final step: force submitFeedback by removing other tools
                    if (step.stepNumber === maxSteps - 1) {
                        this.logger.info(`Final step ${step.stepNumber} - restricting to submitFeedback tool only`);
                        // Add a user message to force the issue and restrict tools
                        const warningMessage = {
                            role: "user",
                            content: [
                                {
                                    type: "text",
                                    text: "**CRITICAL: This is your FINAL step. You MUST call submitFeedback now with your evaluation. No other tools are available.**",
                                },
                            ],
                        };
                        return {
                            ...step,
                            messages: [...step.messages, warningMessage],
                            activeTools: ["submitFeedback"],
                        };
                    }
                    // Warn from halfway point onwards
                    else if (step.stepNumber >= halfwayPoint && remainingSteps > 0) {
                        this.logger.info(`Step ${step.stepNumber} - warning about ${remainingSteps} remaining steps`);
                        const warningMessage = {
                            role: "user",
                            content: [
                                {
                                    type: "text",
                                    text: `**IMPORTANT: You have ${remainingSteps} step${remainingSteps === 1 ? "" : "s"} remaining before you must call submitFeedback. Plan accordingly and wrap up your evaluation.**`,
                                },
                            ],
                        };
                        return {
                            ...step,
                            messages: [...step.messages, warningMessage],
                        };
                    }
                    return step;
                },
                stopWhen: ({ steps }) => {
                    const lastStep = steps[steps.length - 1];
                    const hasSubmitFeedback = lastStep?.toolCalls?.some((tc) => tc.toolName === "submitFeedback");
                    return hasSubmitFeedback ?? false;
                },
                onStepFinish: (stepResult) => {
                    // Extract token usage
                    if (stepResult.usage) {
                        const usage = stepResult.usage;
                        tokenUsage = {
                            input: usage.inputTokens || 0,
                            output: usage.outputTokens || 0,
                            cached: usage.cachedTokens || 0,
                            total: usage.totalTokens || 0,
                        };
                    }
                    if (stepResult.reasoningText) {
                        this.logger.info(`Step finished - reasoning: ${stepResult.reasoningText}`);
                    }
                    if (stepResult.text) {
                        this.logger.info(`Step finished - text: ${stepResult.text}`);
                    }
                },
                onFinish: (finishResult) => {
                    // Extract final token usage
                    if (finishResult.usage) {
                        const usage = finishResult.usage;
                        tokenUsage = {
                            input: usage.inputTokens || 0,
                            output: usage.outputTokens || 0,
                            cached: usage.cachedTokens || 0,
                            total: usage.totalTokens || 0,
                        };
                    }
                    this.logger.info(`Stream finished - finishReason: ${finishResult.finishReason}, hasEvaluation: ${!!evaluation}`);
                },
            });
            // Process stream to extract evaluation
            this.logger.info("Processing stream chunks");
            for await (const chunk of result.fullStream) {
                // Log chunk types for debugging
                if (chunk.type === "tool-call") {
                    const toolName = chunk.toolName;
                    const args = JSON.stringify(chunk.args || {}).slice(0, 200);
                    this.logger.info(`Tool call - ${toolName} with args: ${args}`);
                }
                else if (chunk.type === "tool-result") {
                    const toolName = chunk.toolName;
                    const success = !!chunk.output?.success;
                    const hasError = !!chunk.output?.error;
                    this.logger.info(`Tool result - ${toolName} success=${success} hasError=${hasError}`);
                }
                // Check for tool results to extract evaluation
                if (chunk.type === "tool-result" &&
                    "toolName" in chunk &&
                    chunk.toolName === "submitFeedback") {
                    this.logger.info("Found submitFeedback result");
                    const toolResult = chunk.output;
                    if (toolResult && toolResult.evaluation) {
                        evaluation = toolResult.evaluation;
                        this.logger.info(`Evaluation extracted - Score: ${evaluation.overallScore}, Passed: ${evaluation.passed}`);
                    }
                }
            }
            // Ensure we got an evaluation
            if (!evaluation) {
                // Create a default evaluation if none was submitted
                evaluation = {
                    overallScore: 0,
                    passed: false,
                    criteriaEvaluations: [],
                    summary: "Judge did not submit an evaluation",
                    suggestions: [],
                    tokensUsed: tokenUsage,
                    evaluationDurationMs: Date.now() - startTime,
                };
            }
            else {
                // Add metadata to evaluation
                evaluation.tokensUsed = tokenUsage;
                evaluation.evaluationDurationMs = Date.now() - startTime;
            }
            // Check if evaluation passes threshold
            if (evaluation.overallScore < (this.config.passingThreshold || 70)) {
                this.logger.warn(`Evaluation below passing threshold - score: ${evaluation.overallScore}, threshold: ${this.config.passingThreshold}`);
            }
            return evaluation;
        }
        catch (error) {
            const errorMsg = String(error);
            this.logger.error("Judge evaluation failed", {
                error: {
                    kind: "JudgeEvaluationError",
                    message: errorMsg,
                    stack: error instanceof Error ? error.stack : undefined,
                },
            });
            // Don't crash the process - return a failed evaluation instead
            // This allows the system to continue with other tests
            return {
                overallScore: 0,
                passed: false,
                criteriaEvaluations: [],
                summary: `Judge evaluation failed: ${errorMsg}`,
                suggestions: [
                    errorMsg.includes("import") && errorMsg.includes("module")
                        ? "The evaluation attempted to use ES6 module syntax in browser context. Only plain JavaScript is supported."
                        : "The evaluation encountered an error. Please check the logs for details.",
                ],
                tokensUsed: { input: 0, output: 0, cached: 0, total: 0 },
                evaluationDurationMs: Date.now() - startTime,
                error: errorMsg,
            };
        }
        finally {
            // Cleanup
            try {
                await this.playwrightBridge.cleanup();
            }
            catch (cleanupError) {
                // Log but don't throw cleanup errors
                this.logger.warn(`Cleanup error: ${String(cleanupError)}`);
            }
        }
    }
    /**
     * Builds the user prompt for judge evaluation.
     *
     * @param simulationResult - Simulation to evaluate
     * @param criteria - Evaluation criteria
     * @param appUrl - Application URL
     * @returns Formatted user prompt
     */
    buildUserPrompt(simulationResult, criteria, appUrl) {
        const promptList = simulationResult.stepResults
            .map((sr, idx) => `${idx + 1}. ${sr.prompt}`)
            .join("\n");
        return `Please evaluate the application at: ${appUrl}

The AI agent was given the following prompts:
${promptList}

The agent completed the task in ${simulationResult.duration}ms using:
- Input tokens: ${simulationResult.tokens.input}
- Output tokens: ${simulationResult.tokens.output}
- Total tokens: ${simulationResult.tokens.total}

Evaluation Criteria:

Functional Requirements (40% weight):
${criteria.functionalRequirements.map((r) => `- ${r}`).join("\n")}

UI/UX Requirements (30% weight):
${criteria.uiRequirements.map((r) => `- ${r}`).join("\n")}

Data Requirements (20% weight):
${criteria.dataRequirements.map((r) => `- ${r}`).join("\n")}

${criteria.performanceRequirements?.length
            ? `Performance Requirements (10% weight):\n${criteria.performanceRequirements
                .map((r) => `- ${r}`)
                .join("\n")}`
            : ""}

Instructions:
1. Navigate to the application using the playwright_action tool
2. Systematically test each requirement
3. Capture screenshots as evidence when appropriate
4. Use the submitFeedback tool to submit your final evaluation

Be thorough but efficient. Focus on objective assessment of whether requirements are met.`;
    }
    /**
     * Cleans up resources.
     */
    async cleanup() {
        await this.playwrightBridge.cleanup();
    }
}
//# sourceMappingURL=judge-executor.js.map