import { userPromptToString } from "../../../common.mjs";
import { getDebug } from "@midscene/shared/logger";
import { transformHotkeyInput } from "@midscene/shared/us-keyboard-layout";
import { assert } from "@midscene/shared/utils";
import { actionParser } from "@ui-tars/action-parser";
import { getSummary, getUiTarsPlanningPrompt } from "../../prompt/ui-tars-planning.mjs";
import { AIResponseParseError, callAIWithStringResponse } from "../../service-caller/index.mjs";
import { finalizePixelBbox } from "../../shared/model-locate-result/bbox.mjs";
import { mapLocateResultToPixelBboxByCoordinates } from "../../shared/model-locate-result/pixel-bbox-mapper.mjs";
const debug = getDebug('ui-tars-planning');
const warnLog = getDebug('ui-tars-planning', {
    console: true
});
function pointToLocateParam(point, thought, size) {
    const ctx = {
        preparedSize: size
    };
    const pixelBbox = mapLocateResultToPixelBboxByCoordinates({
        type: 'point',
        coordinates: point
    }, ctx, {
        shape: 'point',
        order: 'xy',
        normalizedBy: 1
    });
    return {
        prompt: thought || '',
        locatedPixelBbox: finalizePixelBbox(pixelBbox, point, ctx)
    };
}
async function uiTarsPlanning(userInstruction, options, uiTarsModelVersion) {
    const { conversationHistory, context, modelRuntime, actionContext } = options;
    const userInstructionText = userPromptToString(userInstruction);
    let instruction = userInstructionText;
    if (actionContext) instruction = `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\n<user_instruction>${userInstructionText}</user_instruction>`;
    const systemPrompt = getUiTarsPlanningPrompt() + instruction;
    const screenshotBase64 = context.screenshot.base64;
    const referenceImageMessages = options.referenceImageMessages ?? [];
    conversationHistory.append({
        role: 'user',
        content: [
            {
                type: 'image_url',
                image_url: {
                    url: screenshotBase64
                }
            }
        ]
    });
    const res = await callAIWithStringResponse([
        {
            role: 'user',
            content: systemPrompt
        },
        ...referenceImageMessages,
        ...conversationHistory.snapshot()
    ], modelRuntime, {
        abortSignal: options.abortSignal
    });
    let convertedText;
    let parsed;
    try {
        convertedText = convertBboxToCoordinates(res.content);
        const { shotSize } = context;
        const parseResult = actionParser({
            prediction: convertedText,
            factor: [
                1000,
                1000
            ],
            screenContext: {
                width: shotSize.width,
                height: shotSize.height
            },
            modelVer: uiTarsModelVersion
        });
        parsed = parseResult.parsed;
    } catch (parseError) {
        const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
        throw new AIResponseParseError(`Parse error: ${errorMessage}`, JSON.stringify(res.content, void 0, 2), res.usage, res.rawChoiceMessage);
    }
    const { shotSize } = context;
    debug('ui-tars modelVer', uiTarsModelVersion, ', parsed', JSON.stringify(parsed));
    const transformActions = [];
    const unhandledActions = [];
    let shouldContinue = true;
    parsed.forEach((action)=>{
        const actionType = (action.action_type || '').toLowerCase();
        if ('click' === actionType) {
            assert(action.action_inputs.start_box, 'start_box is required');
            const point = getPoint(action.action_inputs.start_box);
            const locate = pointToLocateParam(point, action.thought, shotSize);
            transformActions.push({
                type: 'Tap',
                param: {
                    locate
                }
            });
        } else if ('left_double' === actionType) {
            assert(action.action_inputs.start_box, 'start_box is required');
            const point = getPoint(action.action_inputs.start_box);
            const locate = pointToLocateParam(point, action.thought, shotSize);
            transformActions.push({
                type: 'DoubleClick',
                param: {
                    locate
                },
                thought: action.thought || ''
            });
        } else if ('right_single' === actionType) {
            assert(action.action_inputs.start_box, 'start_box is required');
            const point = getPoint(action.action_inputs.start_box);
            const locate = pointToLocateParam(point, action.thought, shotSize);
            transformActions.push({
                type: 'RightClick',
                param: {
                    locate
                },
                thought: action.thought || ''
            });
        } else if ('drag' === actionType) {
            assert(action.action_inputs.start_box, 'start_box is required');
            assert(action.action_inputs.end_box, 'end_box is required');
            const startPoint = getPoint(action.action_inputs.start_box);
            const endPoint = getPoint(action.action_inputs.end_box);
            transformActions.push({
                type: 'DragAndDrop',
                param: {
                    from: pointToLocateParam(startPoint, action.thought, shotSize),
                    to: pointToLocateParam(endPoint, action.thought, shotSize)
                },
                thought: action.thought || ''
            });
        } else if ('type' === actionType) transformActions.push({
            type: 'Input',
            param: {
                value: action.action_inputs.content
            },
            thought: action.thought || ''
        });
        else if ('scroll' === actionType) transformActions.push({
            type: 'Scroll',
            param: {
                direction: action.action_inputs.direction
            },
            thought: action.thought || ''
        });
        else if ('finished' === actionType) {
            shouldContinue = false;
            transformActions.push({
                type: 'Finished',
                param: {},
                thought: action.action_inputs.content || action.thought || ''
            });
        } else if ('hotkey' === actionType) if (action.action_inputs.key) {
            const keys = transformHotkeyInput(action.action_inputs.key);
            transformActions.push({
                type: 'KeyboardPress',
                param: {
                    keyName: keys.join('+')
                },
                thought: action.thought || ''
            });
        } else warnLog('No key found in action: hotkey. Will not perform action.');
        else if ('wait' === actionType) transformActions.push({
            type: 'Sleep',
            param: {
                timeMs: 1000
            },
            thought: action.thought || ''
        });
        else if (actionType) {
            unhandledActions.push({
                type: actionType,
                thought: action.thought || ''
            });
            debug('Unhandled action type:', actionType, 'thought:', action.thought);
        }
    });
    if (0 === transformActions.length) {
        const errorDetails = [];
        if (0 === parsed.length) {
            errorDetails.push('Action parser returned no actions');
            if (res.content.includes('Thought:') && !res.content.includes('Action:')) errorDetails.push('Response contains "Thought:" but missing "Action:" line');
            else errorDetails.push('Response may be malformed or empty');
        }
        if (unhandledActions.length > 0) {
            const types = unhandledActions.map((a)=>a.type).join(', ');
            errorDetails.push(`Unhandled action types: ${types}`);
        }
        const errorMessage = [
            'No actions found in UI-TARS response.',
            ...errorDetails
        ].join('\n');
        throw new AIResponseParseError(errorMessage, JSON.stringify(res.content, void 0, 2), res.usage, res.rawChoiceMessage);
    }
    debug('transformActions', JSON.stringify(transformActions, null, 2));
    const log = getSummary(res.content);
    conversationHistory.append({
        role: 'assistant',
        content: log
    });
    return {
        actions: transformActions,
        log,
        usage: res.usage,
        rawResponse: JSON.stringify(res.content, void 0, 2),
        rawChoiceMessage: res.rawChoiceMessage,
        shouldContinuePlanning: shouldContinue
    };
}
function convertBboxToCoordinates(text) {
    const pattern = /<bbox>(\d+)\s+(\d+)\s+(\d+)\s+(\d+)<\/bbox>/g;
    function replaceMatch(match, x1, y1, x2, y2) {
        const x1Num = Number.parseInt(x1, 10);
        const y1Num = Number.parseInt(y1, 10);
        const x2Num = Number.parseInt(x2, 10);
        const y2Num = Number.parseInt(y2, 10);
        const x = Math.floor((x1Num + x2Num) / 2);
        const y = Math.floor((y1Num + y2Num) / 2);
        return `(${x},${y})`;
    }
    const cleanedText = text.replace(/\[EOS\]/g, '').replace(/```(?:[a-zA-Z0-9_-]+)?/g, '');
    return cleanedText.replace(pattern, replaceMatch).trim();
}
function getPoint(startBox) {
    const [x, y] = JSON.parse(startBox);
    assert('number' == typeof x && Number.isFinite(x) && 'number' == typeof y && Number.isFinite(y), `invalid point data for ui-tars planning: ${startBox}`);
    return [
        x,
        y
    ];
}
export { uiTarsPlanning };

//# sourceMappingURL=planning.mjs.map