import { generateElementByPoint, generateElementByRect } from "@midscene/shared/extractor/dom-util";
import { cropByRect, paddingToMatchBlockByBase64, preProcessImageUrl, scaleImage } from "@midscene/shared/img";
import { getDebug } from "@midscene/shared/logger";
import { assert } from "@midscene/shared/utils";
import { adaptBboxToRect, expandSearchArea, mergeRects } from "../common.mjs";
import { parseAutoGLMLocateResponse } from "./auto-glm/parser.mjs";
import { getAutoGLMLocatePrompt } from "./auto-glm/prompt.mjs";
import { isAutoGLM } from "./auto-glm/util.mjs";
import { extractDataQueryPrompt, parseXMLExtractionResponse, systemPromptToExtract } from "./prompt/extraction.mjs";
import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
import { orderSensitiveJudgePrompt, systemPromptToJudgeOrderSensitive } from "./prompt/order-sensitive-judge.mjs";
import { AIResponseParseError, callAI, callAIWithObjectResponse, callAIWithStringResponse } from "./service-caller/index.mjs";
const debugInspect = getDebug('ai:inspect');
const debugSection = getDebug('ai:section');
const extraTextFromUserPrompt = (prompt)=>{
    if ('string' == typeof prompt) return prompt;
    return prompt.prompt;
};
const promptsToChatParam = async (multimodalPrompt)=>{
    const msgs = [];
    if (multimodalPrompt?.images?.length) {
        msgs.push({
            role: 'user',
            content: [
                {
                    type: 'text',
                    text: 'Next, I will provide all the reference images.'
                }
            ]
        });
        for (const item of multimodalPrompt.images){
            const base64 = await preProcessImageUrl(item.url, !!multimodalPrompt.convertHttpImage2Base64);
            msgs.push({
                role: 'user',
                content: [
                    {
                        type: 'text',
                        text: `this is the reference image named '${item.name}':`
                    }
                ]
            });
            msgs.push({
                role: 'user',
                content: [
                    {
                        type: 'image_url',
                        image_url: {
                            url: base64,
                            detail: 'high'
                        }
                    }
                ]
            });
        }
    }
    return msgs;
};
async function AiLocateElement(options) {
    const { context, targetElementDescription, modelConfig } = options;
    const { modelFamily } = modelConfig;
    const screenshotBase64 = context.screenshot.base64;
    assert(targetElementDescription, "cannot find the target element description");
    const targetElementDescriptionText = extraTextFromUserPrompt(targetElementDescription);
    const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);
    const systemPrompt = isAutoGLM(modelFamily) ? getAutoGLMLocatePrompt(modelFamily) : systemPromptToLocateElement(modelFamily);
    let imagePayload = screenshotBase64;
    let imageWidth = context.shotSize.width;
    let imageHeight = context.shotSize.height;
    let originalImageWidth = imageWidth;
    let originalImageHeight = imageHeight;
    if (options.searchConfig) {
        assert(options.searchConfig.rect, 'searchArea is provided but its rect cannot be found. Failed to locate element');
        assert(options.searchConfig.imageBase64, 'searchArea is provided but its imageBase64 cannot be found. Failed to locate element');
        imagePayload = options.searchConfig.imageBase64;
        imageWidth = options.searchConfig.rect?.width;
        imageHeight = options.searchConfig.rect?.height;
        originalImageWidth = imageWidth;
        originalImageHeight = imageHeight;
    } else if ('qwen2.5-vl' === modelFamily) {
        const paddedResult = await paddingToMatchBlockByBase64(imagePayload);
        imageWidth = paddedResult.width;
        imageHeight = paddedResult.height;
        imagePayload = paddedResult.imageBase64;
    }
    const msgs = [
        {
            role: 'system',
            content: systemPrompt
        },
        {
            role: 'user',
            content: [
                {
                    type: 'image_url',
                    image_url: {
                        url: imagePayload,
                        detail: 'high'
                    }
                },
                {
                    type: 'text',
                    text: isAutoGLM(modelFamily) ? `Tap: ${userInstructionPrompt}` : userInstructionPrompt
                }
            ]
        }
    ];
    if ('string' != typeof targetElementDescription) {
        const addOns = await promptsToChatParam({
            images: targetElementDescription.images,
            convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64
        });
        msgs.push(...addOns);
    }
    if (isAutoGLM(modelFamily)) {
        const { content: rawResponseContent, usage } = await callAIWithStringResponse(msgs, modelConfig);
        debugInspect('auto-glm rawResponse:', rawResponseContent);
        const parsed = parseAutoGLMLocateResponse(rawResponseContent);
        debugInspect('auto-glm thinking:', parsed.think);
        debugInspect('auto-glm coordinates:', parsed.coordinates);
        let resRect;
        let matchedElements = [];
        let errors = [];
        if (parsed.error || !parsed.coordinates) {
            errors = [
                parsed.error || 'Failed to parse auto-glm response'
            ];
            debugInspect('auto-glm parse error:', errors[0]);
        } else {
            const { x, y } = parsed.coordinates;
            debugInspect('auto-glm coordinates [0-999]:', {
                x,
                y
            });
            const pixelX = Math.round(x * imageWidth / 1000);
            const pixelY = Math.round(y * imageHeight / 1000);
            debugInspect('auto-glm pixel coordinates:', {
                pixelX,
                pixelY
            });
            let finalX = pixelX;
            let finalY = pixelY;
            if (options.searchConfig?.rect) {
                finalX += options.searchConfig.rect.left;
                finalY += options.searchConfig.rect.top;
            }
            const element = generateElementByPoint([
                finalX,
                finalY
            ], targetElementDescriptionText);
            resRect = element.rect;
            debugInspect('auto-glm resRect:', resRect);
            if (element) matchedElements = [
                element
            ];
        }
        return {
            rect: resRect,
            parseResult: {
                elements: matchedElements,
                errors
            },
            rawResponse: rawResponseContent,
            usage,
            reasoning_content: parsed.think
        };
    }
    let res;
    try {
        res = await callAIWithObjectResponse(msgs, modelConfig);
    } catch (callError) {
        const errorMessage = callError instanceof Error ? callError.message : String(callError);
        const rawResponse = callError instanceof AIResponseParseError ? callError.rawResponse : errorMessage;
        const usage = callError instanceof AIResponseParseError ? callError.usage : void 0;
        return {
            rect: void 0,
            parseResult: {
                elements: [],
                errors: [
                    `AI call error: ${errorMessage}`
                ]
            },
            rawResponse,
            usage,
            reasoning_content: void 0
        };
    }
    const rawResponse = JSON.stringify(res.content);
    let resRect;
    let matchedElements = [];
    let errors = 'errors' in res.content ? res.content.errors : [];
    try {
        if ('bbox' in res.content && Array.isArray(res.content.bbox) && res.content.bbox.length >= 1) {
            resRect = adaptBboxToRect(res.content.bbox, imageWidth, imageHeight, options.searchConfig?.rect?.left, options.searchConfig?.rect?.top, originalImageWidth, originalImageHeight, modelFamily, options.searchConfig?.scale);
            debugInspect('resRect', resRect);
            const element = generateElementByRect(resRect, targetElementDescriptionText);
            errors = [];
            if (element) matchedElements = [
                element
            ];
        }
    } catch (e) {
        const msg = e instanceof Error ? `Failed to parse bbox: ${e.message}` : 'unknown error in locate';
        if (errors && errors?.length !== 0) errors.push(`(${msg})`);
        else errors = [
            msg
        ];
    }
    return {
        rect: resRect,
        parseResult: {
            elements: matchedElements,
            errors: errors
        },
        rawResponse,
        usage: res.usage,
        reasoning_content: res.reasoning_content
    };
}
async function AiLocateSection(options) {
    const { context, sectionDescription, modelConfig } = options;
    const { modelFamily } = modelConfig;
    const screenshotBase64 = context.screenshot.base64;
    const systemPrompt = systemPromptToLocateSection(modelFamily);
    const sectionLocatorInstructionText = sectionLocatorInstruction(extraTextFromUserPrompt(sectionDescription));
    const msgs = [
        {
            role: 'system',
            content: systemPrompt
        },
        {
            role: 'user',
            content: [
                {
                    type: 'image_url',
                    image_url: {
                        url: screenshotBase64,
                        detail: 'high'
                    }
                },
                {
                    type: 'text',
                    text: sectionLocatorInstructionText
                }
            ]
        }
    ];
    if ('string' != typeof sectionDescription) {
        const addOns = await promptsToChatParam({
            images: sectionDescription.images,
            convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64
        });
        msgs.push(...addOns);
    }
    let result;
    try {
        result = await callAIWithObjectResponse(msgs, modelConfig);
    } catch (callError) {
        const errorMessage = callError instanceof Error ? callError.message : String(callError);
        const rawResponse = callError instanceof AIResponseParseError ? callError.rawResponse : errorMessage;
        const usage = callError instanceof AIResponseParseError ? callError.usage : void 0;
        return {
            rect: void 0,
            imageBase64: void 0,
            error: `AI call error: ${errorMessage}`,
            rawResponse,
            usage
        };
    }
    let sectionRect;
    const sectionBbox = result.content.bbox;
    if (sectionBbox) {
        const targetRect = adaptBboxToRect(sectionBbox, context.shotSize.width, context.shotSize.height, 0, 0, context.shotSize.width, context.shotSize.height, modelFamily);
        debugSection('original targetRect %j', targetRect);
        const referenceBboxList = result.content.references_bbox || [];
        debugSection('referenceBboxList %j', referenceBboxList);
        const referenceRects = referenceBboxList.filter((bbox)=>Array.isArray(bbox)).map((bbox)=>adaptBboxToRect(bbox, context.shotSize.width, context.shotSize.height, 0, 0, context.shotSize.width, context.shotSize.height, modelFamily));
        debugSection('referenceRects %j', referenceRects);
        const mergedRect = mergeRects([
            targetRect,
            ...referenceRects
        ]);
        debugSection('mergedRect %j', mergedRect);
        sectionRect = expandSearchArea(mergedRect, context.shotSize);
        debugSection('expanded sectionRect %j', sectionRect);
    }
    let imageBase64 = screenshotBase64;
    let scale;
    if (sectionRect) {
        const originalWidth = sectionRect.width;
        const originalHeight = sectionRect.height;
        const croppedResult = await cropByRect(screenshotBase64, sectionRect, 'qwen2.5-vl' === modelFamily);
        const scaleRatio = 2;
        const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);
        imageBase64 = scaledResult.imageBase64;
        scale = scaleRatio;
        sectionRect.width = scaledResult.width;
        sectionRect.height = scaledResult.height;
        debugSection('scaled sectionRect from %dx%d to %dx%d (scale=%d)', originalWidth, originalHeight, sectionRect.width, sectionRect.height, scale);
    }
    return {
        rect: sectionRect,
        imageBase64,
        scale,
        error: result.content.error,
        rawResponse: JSON.stringify(result.content),
        usage: result.usage
    };
}
async function AiExtractElementInfo(options) {
    const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } = options;
    const systemPrompt = systemPromptToExtract();
    const screenshotBase64 = context.screenshot.base64;
    const extractDataPromptText = extractDataQueryPrompt(options.pageDescription || '', dataQuery);
    const userContent = [];
    if (extractOption?.screenshotIncluded !== false) userContent.push({
        type: 'image_url',
        image_url: {
            url: screenshotBase64,
            detail: 'high'
        }
    });
    userContent.push({
        type: 'text',
        text: extractDataPromptText
    });
    const msgs = [
        {
            role: 'system',
            content: systemPrompt
        },
        {
            role: 'user',
            content: userContent
        }
    ];
    if (multimodalPrompt) {
        const addOns = await promptsToChatParam({
            images: multimodalPrompt.images,
            convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64
        });
        msgs.push(...addOns);
    }
    const { content: rawResponse, usage, reasoning_content } = await callAI(msgs, modelConfig);
    let parseResult;
    try {
        parseResult = parseXMLExtractionResponse(rawResponse);
    } catch (parseError) {
        const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
        throw new AIResponseParseError(`XML parse error: ${errorMessage}`, rawResponse, usage);
    }
    return {
        parseResult,
        rawResponse,
        usage,
        reasoning_content
    };
}
async function AiJudgeOrderSensitive(description, callAIFn, modelConfig) {
    const systemPrompt = systemPromptToJudgeOrderSensitive();
    const userPrompt = orderSensitiveJudgePrompt(description);
    const msgs = [
        {
            role: 'system',
            content: systemPrompt
        },
        {
            role: 'user',
            content: userPrompt
        }
    ];
    const result = await callAIFn(msgs, modelConfig);
    return {
        isOrderSensitive: result.content.isOrderSensitive ?? false,
        usage: result.usage
    };
}
export { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection };

//# sourceMappingURL=inspect.mjs.map