import { generateElementByRect } from "@midscene/shared/extractor";
import { cropByRect, scaleImage } from "@midscene/shared/img";
import { getDebug } from "@midscene/shared/logger";
import { assert } from "@midscene/shared/utils";
import { expandSearchArea, multimodalPromptToChatMessages, userPromptToMultimodalPrompt, userPromptToString } from "../common.mjs";
import { extractDataQueryPrompt, parseXMLExtractionResponse, systemPromptToExtract } from "./prompt/extraction.mjs";
import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
import { orderSensitiveJudgePrompt, systemPromptToJudgeOrderSensitive } from "./prompt/order-sensitive-judge.mjs";
import { AIResponseParseError, callAI, callAIWithObjectResponse } from "./service-caller/index.mjs";
import { prepareModelImage } from "./workflows/image-preprocess.mjs";
import { mergePixelBboxesToRect, pixelBboxToRect } from "./workflows/inspect/locate-result-rect.mjs";
import { mapSearchAreaPixelBboxToOriginalPixelBbox } from "./workflows/inspect/search-area-mapping.mjs";
const debugInspect = getDebug('ai:inspect');
const debugSection = getDebug('ai:section');
function hasLocateResult(input, resultKey) {
    if (!input || 'object' != typeof input) return false;
    const record = input;
    const locateResult = record[resultKey];
    return Array.isArray(locateResult) ? locateResult.length > 0 : void 0 !== locateResult;
}
async function buildSearchAreaConfig(options) {
    const { context, baseRect } = options;
    const scaleRatio = 2;
    const sectionRect = expandSearchArea(baseRect, context.shotSize);
    const croppedResult = await cropByRect(context.screenshot.base64, sectionRect);
    const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);
    return {
        sourceRect: sectionRect,
        image: {
            imageBase64: scaledResult.imageBase64,
            width: scaledResult.width,
            height: scaledResult.height
        },
        mapping: {
            offset: {
                x: sectionRect.left,
                y: sectionRect.top
            },
            scale: scaleRatio
        }
    };
}
async function AiLocateElement(options) {
    const { targetElementDescription, ...locateOptions } = options;
    const locateAdapter = options.modelRuntime.adapter.locate;
    if ('custom' === locateAdapter.kind) return locateAdapter.locateFn(targetElementDescription, locateOptions);
    return genericLocate(targetElementDescription, locateOptions);
}
async function genericLocate(elementDescription, options) {
    const { context } = options;
    const modelRuntime = options.modelRuntime;
    const { adapter } = modelRuntime;
    assert('standard' === adapter.locate.kind, 'generic locate requires a standard locate adapter');
    const screenshotBase64 = context.screenshot.base64;
    assert(elementDescription, "cannot find the target element description");
    const elementDescriptionText = userPromptToString(elementDescription);
    const userInstructionPrompt = findElementPrompt(elementDescriptionText);
    const systemPrompt = systemPromptToLocateElement(adapter.locate.resultAdapter.promptSpec);
    const modelImage = options.searchConfig?.image ?? {
        imageBase64: screenshotBase64,
        width: context.shotSize.width,
        height: context.shotSize.height
    };
    const preparedImage = await prepareModelImage({
        imageBase64: modelImage.imageBase64,
        width: modelImage.width,
        height: modelImage.height,
        policy: adapter.imagePreprocess
    });
    const imagePayload = preparedImage.imageBase64;
    const msgs = [
        {
            role: 'system',
            content: systemPrompt
        },
        {
            role: 'user',
            content: [
                {
                    type: 'image_url',
                    image_url: {
                        url: imagePayload,
                        detail: 'high'
                    }
                },
                {
                    type: 'text',
                    text: userInstructionPrompt
                }
            ]
        }
    ];
    if ('string' != typeof elementDescription) {
        const addOns = await multimodalPromptToChatMessages(userPromptToMultimodalPrompt(elementDescription));
        msgs.push(...addOns);
    }
    let res;
    try {
        res = await callAIWithObjectResponse(msgs, modelRuntime, {
            abortSignal: options.abortSignal,
            jsonParserSource: 'locate'
        });
    } catch (callError) {
        const errorMessage = callError instanceof Error ? callError.message : String(callError);
        const rawResponse = callError instanceof AIResponseParseError ? callError.rawResponse : errorMessage;
        const usage = callError instanceof AIResponseParseError ? callError.usage : void 0;
        const rawChoiceMessage = callError instanceof AIResponseParseError ? callError.rawChoiceMessage : void 0;
        return {
            rect: void 0,
            parseResult: {
                element: void 0,
                errors: [
                    `AI call error: ${errorMessage}`
                ]
            },
            rawResponse,
            rawChoiceMessage,
            usage,
            reasoning_content: void 0
        };
    }
    const rawResponse = JSON.stringify(res.content);
    let resRect;
    let matchedElement;
    let errors = 'errors' in res.content ? res.content.errors : [];
    const resultAdapter = adapter.locate.resultAdapter;
    if (!hasLocateResult(res.content, resultAdapter.promptSpec.resultKey)) return {
        rect: void 0,
        parseResult: {
            element: void 0,
            errors: errors
        },
        rawResponse,
        rawChoiceMessage: res.rawChoiceMessage,
        usage: res.usage,
        reasoning_content: res.reasoning_content
    };
    try {
        const mapping = options.searchConfig?.mapping;
        const targetPixelBbox = resultAdapter.adaptElementLocateResultToPixelBbox(res.content, {
            preparedSize: preparedImage.preparedSize,
            contentSize: preparedImage.contentSize
        });
        resRect = pixelBboxToRect(mapSearchAreaPixelBboxToOriginalPixelBbox(targetPixelBbox, mapping));
        debugInspect('resRect', resRect);
        const element = generateElementByRect(resRect, elementDescriptionText);
        errors = [];
        if (element) matchedElement = element;
    } catch (e) {
        const msg = e instanceof Error ? `Failed to parse locate result: ${e.message}` : 'unknown error in locate';
        if (errors && errors?.length !== 0) errors.push(`(${msg})`);
        else errors = [
            msg
        ];
    }
    return {
        rect: resRect,
        parseResult: {
            element: matchedElement,
            errors: errors
        },
        rawResponse,
        rawChoiceMessage: res.rawChoiceMessage,
        usage: res.usage,
        reasoning_content: res.reasoning_content
    };
}
async function AiLocateSection(options) {
    const { context, sectionDescription } = options;
    const modelRuntime = options.modelRuntime;
    const { adapter } = modelRuntime;
    assert('standard' === adapter.locate.kind, 'section locate requires a standard locate adapter');
    const screenshotBase64 = context.screenshot.base64;
    const preparedImage = await prepareModelImage({
        imageBase64: screenshotBase64,
        width: context.shotSize.width,
        height: context.shotSize.height,
        policy: adapter.imagePreprocess
    });
    const systemPrompt = systemPromptToLocateSection(adapter.locate.resultAdapter.promptSpec);
    const sectionLocatorInstructionText = sectionLocatorInstruction(userPromptToString(sectionDescription));
    const msgs = [
        {
            role: 'system',
            content: systemPrompt
        },
        {
            role: 'user',
            content: [
                {
                    type: 'image_url',
                    image_url: {
                        url: preparedImage.imageBase64,
                        detail: 'high'
                    }
                },
                {
                    type: 'text',
                    text: sectionLocatorInstructionText
                }
            ]
        }
    ];
    if ('string' != typeof sectionDescription) {
        const addOns = await multimodalPromptToChatMessages(userPromptToMultimodalPrompt(sectionDescription));
        msgs.push(...addOns);
    }
    let result;
    try {
        result = await callAIWithObjectResponse(msgs, modelRuntime, {
            abortSignal: options.abortSignal,
            jsonParserSource: 'section-locator'
        });
    } catch (callError) {
        const errorMessage = callError instanceof Error ? callError.message : String(callError);
        const rawResponse = callError instanceof AIResponseParseError ? callError.rawResponse : errorMessage;
        const usage = callError instanceof AIResponseParseError ? callError.usage : void 0;
        const rawChoiceMessage = callError instanceof AIResponseParseError ? callError.rawChoiceMessage : void 0;
        return {
            searchAreaConfig: void 0,
            error: `AI call error: ${errorMessage}`,
            rawResponse,
            rawChoiceMessage,
            usage
        };
    }
    let searchAreaConfig;
    let sectionError = result.content.error;
    const resultAdapter = adapter.locate.resultAdapter;
    if (!hasLocateResult(result.content, resultAdapter.promptSpec.resultKey)) return {
        searchAreaConfig: void 0,
        error: sectionError,
        rawResponse: JSON.stringify(result.content),
        rawChoiceMessage: result.rawChoiceMessage,
        usage: result.usage
    };
    try {
        const adaptedResult = resultAdapter.adaptSectionLocateResultToPixelBboxGroup(result.content, {
            preparedSize: preparedImage.preparedSize,
            contentSize: preparedImage.contentSize
        });
        const mergedRect = mergePixelBboxesToRect([
            adaptedResult.target,
            ...adaptedResult.references ?? []
        ]);
        debugSection('mergedRect %j', mergedRect);
        const expandedRect = expandSearchArea(mergedRect, context.shotSize);
        const originalWidth = expandedRect.width;
        const originalHeight = expandedRect.height;
        debugSection('expanded sectionRect %j', expandedRect);
        searchAreaConfig = await buildSearchAreaConfig({
            context,
            baseRect: mergedRect
        });
        debugSection('scaled section image from %dx%d to %dx%d (scale=%d)', originalWidth, originalHeight, searchAreaConfig.image.width, searchAreaConfig.image.height, searchAreaConfig.mapping.scale);
    } catch (error) {
        const parseErrorMessage = error instanceof Error ? `Failed to parse section locate result: ${error.message}` : 'unknown error in section locate';
        sectionError = sectionError ? `${sectionError} (${parseErrorMessage})` : parseErrorMessage;
    }
    return {
        searchAreaConfig,
        error: sectionError,
        rawResponse: JSON.stringify(result.content),
        rawChoiceMessage: result.rawChoiceMessage,
        usage: result.usage
    };
}
async function AiExtractElementInfo(options) {
    const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } = options;
    const systemPrompt = systemPromptToExtract({
        screenshotIncluded: extractOption?.screenshotIncluded !== false,
        referenceImagesIncluded: !!multimodalPrompt?.images?.length
    });
    const screenshotBase64 = context.screenshot.base64;
    const extractDataPromptText = extractDataQueryPrompt(options.pageDescription || '', dataQuery);
    const userContent = [];
    if (extractOption?.screenshotIncluded !== false) {
        userContent.push({
            type: 'text',
            text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.'
        });
        userContent.push({
            type: 'image_url',
            image_url: {
                url: screenshotBase64,
                detail: 'high'
            }
        });
    }
    userContent.push({
        type: 'text',
        text: extractDataPromptText
    });
    const msgs = [
        {
            role: 'system',
            content: systemPrompt
        },
        {
            role: 'user',
            content: userContent
        }
    ];
    if (multimodalPrompt) {
        const addOns = await multimodalPromptToChatMessages(multimodalPrompt);
        msgs.push(...addOns);
    }
    const { content: rawResponse, usage, reasoning_content, rawChoiceMessage } = await callAI(msgs, modelRuntime);
    let parseResult;
    try {
        parseResult = parseXMLExtractionResponse(rawResponse);
    } catch (parseError) {
        const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
        throw new AIResponseParseError(`XML parse error: ${errorMessage}`, rawResponse, usage, rawChoiceMessage);
    }
    return {
        parseResult,
        rawResponse,
        rawChoiceMessage,
        usage,
        reasoning_content
    };
}
async function AiJudgeOrderSensitive(description, modelRuntime) {
    const systemPrompt = systemPromptToJudgeOrderSensitive();
    const userPrompt = orderSensitiveJudgePrompt(description);
    const msgs = [
        {
            role: 'system',
            content: systemPrompt
        },
        {
            role: 'user',
            content: userPrompt
        }
    ];
    debugInspect("AiJudgeOrderSensitive: description=%s", description);
    const result = await callAIWithObjectResponse(msgs, modelRuntime, {
        jsonParserSource: 'generic-object'
    });
    return {
        isOrderSensitive: result.content.isOrderSensitive ?? false,
        usage: result.usage
    };
}
export { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, buildSearchAreaConfig, userPromptToString as extraTextFromUserPrompt, genericLocate, multimodalPromptToChatMessages as promptsToChatParam };

//# sourceMappingURL=inspect.mjs.map