/**
 * @license
 * Copyright 2026 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */
/**
 * @fileoverview Tool for visual identification via a single model call.
 *
 * The semantic browser agent uses this tool when it needs to identify
 * elements by visual attributes not present in the accessibility tree
 * (e.g., color, layout, precise coordinates).
 *
 * Unlike the semantic agent which works with the accessibility tree,
 * this tool sends a screenshot to a computer-use model for visual analysis.
 * It returns the model's analysis (coordinates, element descriptions) back
 * to the browser agent, which retains full control of subsequent actions.
 */
import { DeclarativeTool, BaseToolInvocation, Kind, } from '../../tools/tools.js';
import { Environment } from '@google/genai';
import { getVisualAgentModel, isComputerUseModel, } from './modelAvailability.js';
import { debugLogger } from '../../utils/debugLogger.js';
import { LlmRole } from '../../telemetry/llmRole.js';
/**
 * System prompt for the visual analysis model call.
 */
const VISUAL_SYSTEM_PROMPT = `You are a Visual Analysis Agent. You receive a screenshot of a browser page and an instruction.

Your job is to ANALYZE the screenshot and provide precise information that a browser automation agent can act on.

COORDINATE SYSTEM:
- Coordinates are pixel-based relative to the viewport
- (0,0) is top-left of the visible area
- Estimate element positions from the screenshot

RESPONSE FORMAT:
- For coordinate identification: provide exact (x, y) pixel coordinates
- For element identification: describe the element's visual location and appearance
- For layout analysis: describe the spatial relationships between elements
- Be concise and actionable — the browser agent will use your response to decide what action to take

IMPORTANT:
- You are NOT performing actions — you are only providing visual analysis
- Include coordinates when possible so the caller can use click_at(x, y)
- If the element is not visible in the screenshot, say so explicitly`;
/**
 * Invocation for the analyze_screenshot tool.
 * Makes a single generateContent call with a screenshot.
 */
class AnalyzeScreenshotInvocation extends BaseToolInvocation {
    browserManager;
    config;
    constructor(browserManager, config, params, messageBus) {
        super(params, messageBus, 'analyze_screenshot', 'Analyze Screenshot');
        this.browserManager = browserManager;
        this.config = config;
    }
    getDescription() {
        const instruction = String(this.params['instruction'] ?? '');
        return `Visual analysis: "${instruction}"`;
    }
    async execute(signal) {
        try {
            const instruction = String(this.params['instruction'] ?? '');
            debugLogger.log(`Visual analysis requested: ${instruction}`);
            // Capture screenshot via MCP tool
            const screenshotResult = await this.browserManager.callTool('take_screenshot', {});
            // Extract base64 image data from MCP response.
            // Search ALL content items for image type — MCP returns [text, image]
            // where content[0] is a text description and content[1] is the actual PNG.
            let screenshotBase64 = '';
            let mimeType = 'image/png';
            if (screenshotResult.content && Array.isArray(screenshotResult.content)) {
                for (const item of screenshotResult.content) {
                    if (item.type === 'image' && item.data) {
                        screenshotBase64 = item.data;
                        mimeType = item.mimeType ?? 'image/png';
                        break;
                    }
                }
            }
            if (!screenshotBase64) {
                return {
                    llmContent: 'Failed to capture screenshot for visual analysis. Use accessibility tree elements instead.',
                    returnDisplay: 'Screenshot capture failed',
                    error: { message: 'Screenshot capture failed' },
                };
            }
            // Make a single generateContent call with the visual model
            const visualModel = getVisualAgentModel(this.config);
            const contentGenerator = this.config.getContentGenerator();
            // Computer-use models require the computerUse tool declaration in every
            // request. We exclude all predefined action functions so the model
            // provides text analysis rather than issuing actions.
            // Non-computer-use models (e.g., gemini-2.0-flash) do plain text
            // analysis natively and don't need this declaration.
            const tools = isComputerUseModel(visualModel)
                ? [
                    {
                        computerUse: {
                            environment: Environment.ENVIRONMENT_BROWSER,
                            excludedPredefinedFunctions: [
                                'open_web_browser',
                                'click_at',
                                'key_combination',
                                'drag_and_drop',
                            ],
                        },
                    },
                ]
                : undefined;
            const response = await contentGenerator.generateContent({
                model: visualModel,
                config: {
                    temperature: 0,
                    topP: 0.95,
                    systemInstruction: VISUAL_SYSTEM_PROMPT,
                    abortSignal: signal,
                    ...(tools ? { tools } : {}),
                },
                contents: [
                    {
                        role: 'user',
                        parts: [
                            {
                                text: `Analyze this screenshot and respond to the following instruction:\n\n${instruction}`,
                            },
                            {
                                inlineData: {
                                    mimeType,
                                    data: screenshotBase64,
                                },
                            },
                        ],
                    },
                ],
            }, 'visual-analysis', LlmRole.UTILITY_TOOL);
            // Extract response content. Computer-use models may still return
            // functionCall parts even with exclusions, so we handle both text
            // and functionCall parts defensively.
            const parts = response.candidates?.[0]?.content?.parts ?? [];
            const textParts = parts.filter((p) => p.text).map((p) => p.text);
            const functionCallParts = parts
                .filter((p) => p.functionCall)
                .map((p) => {
                const fc = p.functionCall;
                const argsStr = fc.args ? JSON.stringify(fc.args) : '';
                return `Action: ${fc.name}${argsStr ? ` with args ${argsStr}` : ''}`;
            });
            const responseText = [...textParts, ...functionCallParts].join('\n');
            if (!responseText) {
                return {
                    llmContent: 'Visual model returned no analysis. Use accessibility tree elements instead.',
                    returnDisplay: 'Visual analysis returned empty response',
                    error: { message: 'Empty visual analysis response' },
                };
            }
            debugLogger.log(`Visual analysis complete: ${responseText}`);
            return {
                llmContent: `Visual Analysis Result:\n${responseText}`,
                returnDisplay: `Visual Analysis Result:\n${responseText}`,
            };
        }
        catch (error) {
            const errorMsg = error instanceof Error ? error.message : String(error);
            debugLogger.error(`Visual analysis failed: ${errorMsg}`);
            // Provide a graceful fallback message for model unavailability
            const isModelError = errorMsg.includes('404') ||
                errorMsg.includes('403') ||
                errorMsg.includes('not found') ||
                errorMsg.includes('permission');
            const fallbackMsg = isModelError
                ? 'Visual analysis model is not available. Use accessibility tree elements (uids from take_snapshot) for all interactions instead.'
                : `Visual analysis failed: ${errorMsg}. Use accessibility tree elements instead.`;
            return {
                llmContent: fallbackMsg,
                returnDisplay: fallbackMsg,
                error: { message: errorMsg },
            };
        }
    }
}
/**
 * DeclarativeTool for screenshot-based visual analysis.
 */
class AnalyzeScreenshotTool extends DeclarativeTool {
    browserManager;
    config;
    constructor(browserManager, config, messageBus) {
        super('analyze_screenshot', 'analyze_screenshot', 'Analyze the current page visually using a screenshot. Use when you need to identify elements by visual attributes (color, layout, position) not available in the accessibility tree, or when you need precise pixel coordinates for click_at. Returns visual analysis — you perform the actions yourself.', Kind.Other, {
            type: 'object',
            properties: {
                instruction: {
                    type: 'string',
                    description: 'What to identify or analyze visually (e.g., "Find the coordinates of the blue submit button", "What is the layout of the navigation menu?").',
                },
            },
            required: ['instruction'],
        }, messageBus, true, // isOutputMarkdown
        false);
        this.browserManager = browserManager;
        this.config = config;
    }
    build(params) {
        return new AnalyzeScreenshotInvocation(this.browserManager, this.config, params, this.messageBus);
    }
}
/**
 * Creates the analyze_screenshot tool for the browser agent.
 */
export function createAnalyzeScreenshotTool(browserManager, config, messageBus) {
    return new AnalyzeScreenshotTool(browserManager, config, messageBus);
}
//# sourceMappingURL=analyzeScreenshot.js.map