import * as fs from 'fs';
import * as path from 'path';
import { SelectorResolver } from '../core/selectorResolver.js';
import { Logger } from '../utils/logger.js';
import { Healer } from '../utils/healer.js';
import { CaptchaSolver } from '../utils/captchaSolver.js';
import { DownloadHandler } from '../utils/downloadHandler.js';
import { S3Uploader } from '../integrations/s3Uploader.js';
import { VariableResolver } from '../utils/variableResolver.js';
/**
 * Executes DSL-based flows and validations.
 */
export class DSLExecutor {
    static DEFAULT_TIMEOUT = 30000;
    static attachedPages = new Set();
    static aiConfig = null;
    static setGlobalTimeout(ms) {
        this.DEFAULT_TIMEOUT = ms;
    }
    static setAIConfig(config) {
        this.aiConfig = config;
    }
    /**
     * Attaches a diagnostic listener to capture interesting XHR/Fetch traffic.
     */
    static attachNetworkListener(page) {
        if (this.attachedPages.has(page))
            return;
        this.attachedPages.add(page);
        page.on('response', async (response) => {
            try {
                const request = response.request();
                const url = response.url();
                const status = response.status();
                const method = request.method();
                // Log all POST requests, auth endpoints, and any errors
                const isAuth = url.includes('login') || url.includes('auth') || url.includes('signin') || url.includes('signup');
                if (method === 'POST' || status >= 400 || isAuth) {
                    const contentType = response.headers()['content-type'] || '';
                    let responseBody = '[NON-TEXT-BODY]';
                    if (contentType.includes('json') || contentType.includes('text') || contentType.includes('javascript')) {
                        responseBody = await response.text().catch(() => '[FAILED_TO_READ_BODY]');
                    }
                    const requestBody = request.postData() || '[NO-PAYLOAD]';
                    Logger.log(`
┌── [NETWORK ACTIVITY] ${method} ${status} ──────────────────
│ URL:
│ ${url}
│
│ PAYLOAD:
│ ${requestBody}
│
│ RESPONSE:
│ ${responseBody}
└───────────────────────────────────────────────────────────`);
                }
            }
            catch (e) {
                Logger.error("exception during network interception : ", e);
            }
        });
    }
    /**
     * Attaches a listener to capture frontend console logs and errors.
     */
    static attachConsoleListener(page) {
        // Console events (log, warn, error)
        page.on('console', message => {
            const type = message.type();
            if (type === 'error' || type === 'warning') {
                Logger.log(`[Console] ${type.toUpperCase()}: ${message.text()}`);
            }
        });
        // Uncaught exceptions
        page.on('pageerror', error => {
            Logger.log(`[PageError] UNCAUGHT EXCEPTION: ${error.message}\n${error.stack}`);
        });
    }
    /**
     * Executes a series of DSL steps.
     */
    static async executeFlow(context, steps, onStep) {
        const { page } = context;
        if (!page || !steps) {
            throw new Error(`Cannot execute flow. Page or steps missing.`);
        }
        // Diagnostic logging only for validation mode to avoid log bloat in large scraping runs
        if (context.scenario.mode === 'validate') {
            this.attachNetworkListener(page);
            this.attachConsoleListener(page);
        }
        for (let i = 0; i < steps.length; i++) {
            if (context.isStopped) {
                throw new Error('STOP_SIGNAL: Execution halted by user');
            }
            const step = steps[i];
            if (onStep)
                await onStep(step, i);
            if (context.onProgress)
                await context.onProgress(step, i);
            await this.executeStep(context, step, i, steps.length);
        }
    }
    static async executeStep(context, step, index, total) {
        const { inputs } = context;
        let value = step.value;
        if (step.inputKey && inputs[step.inputKey] !== undefined && inputs[step.inputKey] !== null) {
            value = String(inputs[step.inputKey]);
            Logger.log(`[Executor] Resolved inputKey "${step.inputKey}" to "${value}"`);
        }
        // Resolve variables in value
        if (typeof value === 'string' || value === null) {
            value = VariableResolver.resolve(value, context);
        }
        Logger.log(`[Executor] Step ${index + 1}/${total}: ${step.action} on ${step.target || 'N/A'}`);
        try {
            await this.executeAction(context, step, value);
        }
        catch (error) {
            if (step.optional) {
                Logger.warn(`Optional step failed, continuing: ${error.message}`);
            }
            else {
                throw error;
            }
        }
    }
    /**
     * Resolves {{variable}} patterns from context.store
     */
    /**
     * Normalizes keys to lower_snake_case
     */
    static sanitizeKey(text) {
        if (!text)
            return '';
        return text
            .trim()
            .toLowerCase()
            .replace(/[^a-z0-9]+/g, '_')
            .replace(/^_+|_+$/g, '');
    }
    /**
     * Strips special characters from the beginning and end of values
     */
    static sanitize(text) {
        if (!text)
            return '';
        // Preserve internal formatting but trim external junk
        return text.trim().replace(/^[^a-zA-Z0-9$]+|[^a-zA-Z0-9!.]+$/g, '').trim();
    }
    /**
     * Performs validation based on expected result.
     */
    static async validate(context) {
        const { page, expected } = context;
        if (!page || !expected) {
            Logger.warn('Validation skipped: page or expected result missing.');
            return;
        }
        Logger.log(`Validating outcome: expected ${expected.type}`);
        // Resolve validation targets (e.g. from a 'shared.validation' registry entry or similar)
        // For now, we assume validation targets are passed in metadata or resolved from a standard key
        const successTargetKey = context.metadata?.successTarget || 'shared.success_indicator';
        const errorTargetKey = context.metadata?.errorTarget || 'shared.error_message';
        if (expected.type === 'success') {
            const selector = SelectorResolver.resolve(successTargetKey);
            const count = await page.locator(selector).count();
            if (count === 0) {
                throw new Error(`Validation Failed: Expected success indicator (${successTargetKey}) not found.`);
            }
        }
        else if (expected.type === 'error') {
            const selector = SelectorResolver.resolve(errorTargetKey);
            const text = await page.textContent(selector);
            if (!text?.includes(expected.message || '')) {
                throw new Error(`Validation Failed: Expected error mismatch. \nExpected to include: "${expected.message}"\nActual text: "${text}"`);
            }
        }
        Logger.log('Validation passed successfully!');
    }
    /**
     * Executes a single DSL action.
     */
    static async executeAction(context, step, value) {
        const { page } = context;
        if (!page)
            throw new Error('Page is missing in ExecutionContext');
        // SCHEMA TRANSLATION: Support both DSL (action/target) and Recorder (type/selector) formats
        const action = step.action || step.type;
        let target = step.target || step.selector;
        if (!action)
            throw new Error(`Missing action type in step: ${JSON.stringify(step)}`);
        const NO_TARGET_ACTIONS = ['navigate', 'sleep', 'network_idle', 'humanize', 'solve_captcha'];
        if (!target && !NO_TARGET_ACTIONS.includes(action))
            throw new Error(`Missing target/selector in step: ${JSON.stringify(step)}`);
        // Resolve variables in target and value
        target = VariableResolver.resolve(target, context);
        if (value !== undefined) {
            value = VariableResolver.resolve(value, context);
        }
        // SPECIAL CASE: Navigate doesn't need selector resolution
        if (action === 'navigate') {
            const url = value || target;
            if (!url)
                throw new Error('URL required for navigate action');
            // Ensure stealth is applied BEFORE navigation to avoid leaking "HeadlessChrome"
            await this.applyStealth(page);
            Logger.log(`Action: navigate to ${url}`);
            await Promise.all([
                page.goto(url, { waitUntil: 'networkidle' }),
                page.waitForLoadState('load')
            ]);
            return;
        }
        if (action === 'humanize') {
            Logger.log('Action: humanize (Applying Deep Stealth & Interaction patterns)');
            await this.applyStealth(page);
            // CRITICAL: Stealth scripts (addInitScript) only apply to NEW document loads.
            // We must reload the current page to ensure reCAPTCHA/Analytics restart with the masked identity.
            Logger.log('Reloading page to activate stealth environment...');
            await page.reload({ waitUntil: 'networkidle' });
            // 3. Realistic Mouse movements (bezier-like or jittery)
            Logger.log('Simulating human interaction patterns (Mouse/Scroll)...');
            for (let i = 0; i < 12; i++) {
                const x = Math.floor(Math.random() * 1000) + 100;
                const y = Math.floor(Math.random() * 800) + 100;
                await page.mouse.move(x, y, { steps: 25 });
                await page.waitForTimeout(Math.random() * 300 + 100);
            }
            // 4. Natural Scrolling (accelerating/decelerating)
            await page.evaluate(async () => {
                for (let i = 0; i < 3; i++) {
                    const distance = Math.floor(Math.random() * 500) + 200;
                    for (let j = 0; j < distance; j += 25) {
                        window.scrollBy(0, 25);
                        await new Promise(r => setTimeout(r, 15));
                    }
                    await new Promise(r => setTimeout(r, 500));
                    for (let j = 0; j < distance; j += 25) {
                        window.scrollBy(0, -25);
                        await new Promise(r => setTimeout(r, 10));
                    }
                }
            });
            // 5. Interaction Warmup: Wait for reCAPTCHA to "observe" us
            await page.waitForTimeout(3000);
            return;
        }
        if (action === 'solve_captcha_service') {
            Logger.log('Action: solve_captcha_service (External Solver)');
            const settings = context.settings;
            const solverKey = settings?.captcha?.twocaptcha?.apiKey || settings?.captcha?.anticaptcha?.apiKey;
            const provider = settings?.captcha?.twocaptcha?.apiKey ? '2captcha' : 'anticaptcha';
            if (!solverKey) {
                Logger.warn('[Captcha] No Solver API Key found in settings. Skipping step.');
                return;
            }
            // Detection
            const siteKey = await page.evaluate(() => {
                const el = document.querySelector('.g-recaptcha, [data-sitekey], iframe[src*="recaptcha"]');
                if (!el)
                    return null;
                return el.getAttribute('data-sitekey') || new URL(el.getAttribute('src') || '').searchParams.get('k');
            });
            if (!siteKey) {
                Logger.warn('[Captcha] No sitekey found on page. Skipping external solve.');
                return;
            }
            try {
                Logger.log(`[Captcha] Using ${provider} to solve sitekey: ${siteKey}`);
                const token = await CaptchaSolver.solve(provider, solverKey, siteKey, page.url());
                // Injection
                await page.evaluate((token) => {
                    // 1. Fill standard hidden input
                    const inputs = document.querySelectorAll('[name="g-recaptcha-response"], #g-recaptcha-response');
                    inputs.forEach(i => i.value = token);
                    // 2. Call callback if exists
                    // @ts-ignore
                    if (window.___grecaptcha_cfg && window.___grecaptcha_cfg.clients) {
                        // @ts-ignore
                        const clients = window.___grecaptcha_cfg.clients;
                        for (const id in clients) {
                            const client = clients[id];
                            for (const key in client) {
                                if (client[key] && client[key].callback) {
                                    if (typeof client[key].callback === 'function') {
                                        client[key].callback(token);
                                    }
                                    else if (typeof client[key].callback === 'string') {
                                        // @ts-ignore
                                        if (window[client[key].callback])
                                            window[client[key].callback](token);
                                    }
                                }
                            }
                        }
                    }
                }, token);
                Logger.log('[Captcha] Token injected successfully.');
            }
            catch (err) {
                Logger.error('[Captcha] Service solving failed', err);
                if (!step.optional)
                    throw err;
            }
            return;
        }
        if (action === 'solve_captcha') {
            Logger.log('Action: solve_captcha (Attempting bypass)');
            const currentUrl = page.url();
            if (currentUrl.includes('recaptcha') || (await page.content()).includes('recaptcha')) {
                Logger.log('[Captcha] reCAPTCHA detected. Performing humanized interaction sequence...');
                // Humanized "thinking" movements
                for (let i = 0; i < 5; i++) {
                    await page.mouse.move(Math.random() * 500, Math.random() * 500, { steps: 10 });
                    await page.mouse.wheel(0, (Math.random() - 0.5) * 100);
                    await page.waitForTimeout(200 + Math.random() * 300);
                }
                // Look for sitekey
                const siteKey = await page.evaluate(() => {
                    const el = document.querySelector('.g-recaptcha, [data-sitekey], iframe[src*="recaptcha"]');
                    if (!el)
                        return null;
                    return el.getAttribute('data-sitekey') || new URL(el.getAttribute('src') || '').searchParams.get('k');
                });
                if (siteKey) {
                    Logger.log(`[Captcha] Sitekey found: ${siteKey}. Interaction quality increased.`);
                }
            }
            await page.waitForTimeout(3000);
            return;
        }
        // RESOLVE SELECTOR
        const selector = SelectorResolver.resolve(target);
        let locator = step.frame
            ? page.frameLocator(SelectorResolver.resolve(step.frame)).locator(selector)
            : page.locator(selector);
        // Enhanced logging for interaction values
        let logMsg = `Action: ${action} on ${target} (${selector}) ${step.frame ? `in frame ${step.frame}` : ''}`;
        if (action === 'type' || action === 'select') {
            logMsg += ` -> value: "${value}"`;
        }
        else if (action === 'check') {
            logMsg += ` -> setting state: checked`;
        }
        else if (action === 'uncheck') {
            logMsg += ` -> setting state: unchecked`;
        }
        Logger.log(logMsg, { value });
        // HOVER BEFORE INTERACTION (Humanize improvement for anti-bot)
        const interactionActions = ['click', 'type', 'select', 'hover', 'dispatch_event'];
        const actionTimeout = step.timeout || (step.optional ? 5000 : this.DEFAULT_TIMEOUT);
        if (interactionActions.includes(action)) {
            try {
                // Move mouse to element first with natural timing to satisfy interaction patterns
                await locator.first().hover({ timeout: Math.min(actionTimeout, 2000) }).catch(() => { });
                // Add a "deliberation" delay (humans don't click instantly after moving)
                await page.waitForTimeout(Math.random() * 500 + 200);
            }
            catch (e) {
                // Fallback to healer if hover fails (likely target missing)
                // Resolve all possible candidate selectors from the centralized registry
                const candidates = SelectorResolver.resolveMany(target);
                const healed = await Healer.heal(page, candidates, this.aiConfig || undefined, { action, target, error: e.message });
                if (healed) {
                    locator = healed;
                }
                else {
                    throw e;
                }
            }
        }
        const perform = async (loc) => {
            const options = { force: true, timeout: actionTimeout };
            switch (action) {
                case 'click':
                    await loc.click(options);
                    // If it's a login/submit button, give it a moment to process redirection
                    if (target.toLowerCase().includes('login') || target.toLowerCase().includes('submit')) {
                        await page.waitForTimeout(2000);
                    }
                    break;
                case 'type':
                    if (value === undefined)
                        throw new Error(`Value required for 'type' action on ${target}`);
                    await loc.click(options); // Ensure focus
                    // HUMANIZED TYING: Randomize delay between characters to avoid fixed-cadence detection
                    await loc.pressSequentially(value, {
                        delay: Math.floor(Math.random() * 100) + 50,
                        timeout: actionTimeout
                    });
                    break;
                case 'select':
                    if (value === undefined)
                        throw new Error(`Value required for 'select' action on ${target}`);
                    Logger.log(`Action: select on ${step.inputKey || target} (${target})  -> value: "${value}"`, { value });
                    try {
                        // Explicit click to trigger dynamic dropdown listeners (Drupal/React/Vue patterns)
                        await loc.click({ force: true, timeout: 2000 }).catch(() => { });
                        await loc.waitFor({ state: 'visible', timeout: 3000 }).catch(() => { });
                    }
                    catch (e) {
                        // Ignore pre-interaction failures; selectOption is the primary goal
                    }
                    await loc.selectOption(value, { timeout: actionTimeout });
                    break;
                case 'check':
                    await loc.check(options);
                    break;
                case 'uncheck':
                    await loc.uncheck(options);
                    break;
                case 'hover':
                    await loc.hover(options);
                    break;
                case 'right_click':
                    await loc.click({ ...options, button: 'right' });
                    break;
                case 'scroll_into_view':
                    await loc.scrollIntoViewIfNeeded({ timeout: actionTimeout });
                    break;
                case 'drag_and_drop':
                    if (!value)
                        throw new Error(`Drop target selector required in 'value' for drag_and_drop on ${target}`);
                    const dropTarget = page.locator(SelectorResolver.resolve(value));
                    await loc.dragTo(dropTarget, { force: true });
                    break;
                case 'upload':
                    if (!value)
                        throw new Error(`File path required in 'value' for 'upload' action on ${target}`);
                    await loc.setInputFiles(value);
                    break;
                case 'dbClick':
                case 'double_click':
                    await loc.dblclick({ force: true });
                    break;
                case 'wait':
                    const state = (value === 'hidden' || value === 'detached') ? value : 'visible';
                    const timeout = step.timeout || this.DEFAULT_TIMEOUT;
                    await loc.first().waitFor({ state, timeout });
                    break;
                case 'regex':
                case 'exists':
                case 'validate': {
                    const targetLoc = loc.first();
                    await page.waitForTimeout(500); // Stability buffer for SPA rendering
                    const validationTimeout = step.timeout || this.DEFAULT_TIMEOUT;
                    await targetLoc.waitFor({ state: 'visible', timeout: validationTimeout });
                    const count = await targetLoc.count();
                    if (count === 0)
                        throw new Error(`Validation Failed: Target not found or not visible: ${target}`);
                    const actualText = (step.attribute ? await targetLoc.getAttribute(step.attribute) : await targetLoc.textContent()) || '';
                    if (action === 'regex' || step.regex) {
                        const pattern = step.regex || value;
                        if (!pattern)
                            throw new Error(`Regex pattern required for 'regex' action on ${target}`);
                        const re = new RegExp(pattern);
                        if (!re.test(actualText.trim())) {
                            throw new Error(`Regex Validation Failed on ${target}: Expected pattern "${pattern}", but found "${actualText.trim()}"`);
                        }
                        Logger.log(`Regex Validation Successful: ${target} matches pattern "${pattern}"`);
                    }
                    else if (step.attribute) {
                        if (actualText !== value) {
                            throw new Error(`Validation Failed on ${target} [${step.attribute}]: Expected "${value}", but found "${actualText}"`);
                        }
                        Logger.log(`Validation Successful: ${target} [${step.attribute}] is "${value}"`);
                    }
                    else if (value && value.trim() !== "") {
                        if (!actualText?.includes(value)) {
                            throw new Error(`Validation Failed on ${target}: Expected text to include "${value}", but found "${actualText?.trim()}"`);
                        }
                        Logger.log(`Validation Successful: ${target} contains "${value}"`);
                    }
                    else {
                        Logger.log(`Validation Successful: ${target} exists and is visible.`);
                    }
                    break;
                }
                case 'hover':
                    await locator.first().hover();
                    break;
                case 'sleep':
                    const ms = parseInt(value || '1000');
                    Logger.log(`Sleeping for ${ms}ms...`);
                    await page.waitForTimeout(ms);
                    break;
                case 'network_idle':
                    Logger.log('Waiting for network idle...');
                    await page.waitForLoadState('networkidle', { timeout: 60000 });
                    break;
                case 'dispatch_event':
                    const eventName = step.attribute || 'change';
                    let eventPayload = { bubbles: true };
                    if (value) {
                        try {
                            eventPayload = { ...eventPayload, ...JSON.parse(value) };
                        }
                        catch (e) {
                            // If it's just a raw string instead of JSON object, you can set it directly
                            // but usually dispatchEvent needs an object or it ignores it
                        }
                    }
                    Logger.log(`Dispatching event: ${eventName} on ${target}`);
                    await loc.dispatchEvent(eventName, eventPayload);
                    break;
                case 'new_tab':
                    await loc.click();
                    break;
                case 'waitForDownload':
                    Logger.log('Waiting for download event...');
                    const downloadPromise = page.waitForEvent('download');
                    context.store.set('downloadPromise', downloadPromise);
                    break;
                case 'solve_captcha_api': {
                    if (!step.captchaConfig)
                        throw new Error('solve_captcha_api requires captchaConfig');
                    const config = step.captchaConfig;
                    Logger.log(`[Captcha] Solving captcha for ${target}...`);
                    // 1. Capture captcha element screenshot
                    const captchaLoc = loc.first();
                    const buffer = await captchaLoc.screenshot();
                    let taskId = '';
                    const headers = {};
                    (config.headers || []).forEach((h) => { headers[h.name] = h.value; });
                    // 2. Submit to solver API
                    try {
                        const submitUrl = `${config.baseUrl}${config.submitPath}`;
                        Logger.log(`[Captcha] Submitting to solver API: ${submitUrl}`);
                        let res;
                        if (config.submitType === 'base64') {
                            const base64 = buffer.toString('base64');
                            const body = { [config.submitParam]: base64 };
                            Logger.log(`[Captcha] Payload (Base64): ${JSON.stringify({ [config.submitParam]: 'base64_data...' })}`);
                            res = await fetch(submitUrl, {
                                method: 'POST',
                                headers: { ...headers, 'Content-Type': 'application/json' },
                                body: JSON.stringify(body)
                            });
                        }
                        else {
                            // Multipart/form-data
                            const formData = new FormData();
                            const blob = new Blob([new Uint8Array(buffer)], { type: 'image/png' });
                            formData.append(config.submitParam, blob, 'captcha.png');
                            Logger.log(`[Captcha] Payload: Multipart/form-data (${config.submitParam}=captcha.png)`);
                            res = await fetch(submitUrl, {
                                method: 'POST',
                                headers,
                                body: formData
                            });
                        }
                        const data = await res.json();
                        Logger.log(`[Captcha] Submit Response [${res.status}]: ${JSON.stringify(data)}`);
                        taskId = data[config.submitResponseKey];
                    }
                    catch (e) {
                        throw new Error(`Captcha Submission Failed: ${e.message}`);
                    }
                    if (!taskId)
                        throw new Error('Captcha API did not return a Task ID');
                    Logger.log(`[Captcha] Task submitted. ID: ${taskId}. Polling for result...`);
                    // 3. Poll for result
                    let solution = '';
                    let attempts = 0;
                    const maxAttempts = 15; // 30 seconds total
                    while (attempts < maxAttempts) {
                        await new Promise(r => setTimeout(r, 2000));
                        try {
                            const statusUrl = `${config.baseUrl}${config.statusPath.replace('{id}', taskId)}`;
                            Logger.log(`[Captcha] Checking status: ${statusUrl}`);
                            const res = await fetch(statusUrl, { headers });
                            const data = await res.json();
                            Logger.log(`[Captcha] Status Response [${res.status}]: ${JSON.stringify(data)}`);
                            if (data[config.statusCheckKey] === config.statusSuccessValue) {
                                solution = data[config.statusResultKey];
                                break;
                            }
                        }
                        catch (e) {
                            Logger.warn(`[Captcha] Polling error: ${e.message}`);
                        }
                        attempts++;
                    }
                    if (!solution)
                        throw new Error('Captcha solving timed out');
                    Logger.log(`[Captcha] Solved! Text: ${solution}`);
                    // Store result in context
                    context.store.set('captcha_text', solution);
                    // 4. Fill input field if target provided
                    if (config.inputTarget) {
                        const inputLoc = page.locator(SelectorResolver.resolve(config.inputTarget)).first();
                        await inputLoc.fill(solution);
                        Logger.log(`[Captcha] Populated ${config.inputTarget} with solved text.`);
                    }
                    break;
                }
                case 'captureDownload':
                    const promise = context.store.get('downloadPromise');
                    if (!promise)
                        throw new Error('captureDownload called without a previous waitForDownload');
                    const download = await promise;
                    const filePath = await DownloadHandler.capture(download);
                    context.store.set('downloadedFilePath', filePath);
                    context.store.set('fileName', download.suggestedFilename());
                    break;
                case 'download':
                    // Unified action: Click -> Wait -> Capture -> S3
                    Logger.log(`Executing unified download for ${target}...`);
                    const dlPromise = page.waitForEvent('download');
                    await loc.click({ force: true });
                    const dl = await dlPromise;
                    const savedPath = await DownloadHandler.capture(dl);
                    const s3Result = await S3Uploader.uploadFile(savedPath, context.metadata);
                    context.store.set('fileUrl', s3Result.url);
                    DownloadHandler.cleanup(savedPath);
                    Logger.log(`Unified download complete. S3 URL: ${s3Result.url}`);
                    break;
                case 'uploadToS3':
                    const s3Path = context.store.get('downloadedFilePath');
                    if (!s3Path)
                        throw new Error('uploadToS3 called without a captured file');
                    const uploadResult = await S3Uploader.uploadFile(s3Path, context.metadata);
                    context.store.set('fileUrl', uploadResult.url);
                    DownloadHandler.cleanup(s3Path);
                    break;
                case 'scrape':
                    const scrapeResults = context.scenario.isPreview ? [(await loc.first().textContent()) || ''] : await loc.allTextContents();
                    const cleaned = scrapeResults.map(t => t.trim()).filter(Boolean);
                    const storeKey = this.sanitize(step.inputKey || target);
                    const finalVal = cleaned.length === 1 ? cleaned[0] : cleaned;
                    context.store.set(storeKey, finalVal);
                    if (context.isBuffering) {
                        context.extractionBuffer[storeKey] = finalVal;
                    }
                    Logger.log(`Scraped ${cleaned.length} items for ${storeKey} ${context.scenario.isPreview ? '(Preview Limit: 1)' : ''}`);
                    break;
                case 'scrapeAttr':
                    const attrName = value || 'href';
                    const attrVal = await loc.first().getAttribute(attrName);
                    const attrStoreKey = this.sanitize(step.inputKey || target);
                    const finalAttrVal = attrVal ? attrVal.trim() : '';
                    context.store.set(attrStoreKey, finalAttrVal);
                    if (context.isBuffering) {
                        context.extractionBuffer[attrStoreKey] = finalAttrVal;
                    }
                    Logger.log(`Scraped attribute ${attrName}: ${attrVal} -> ${attrStoreKey}`);
                    break;
                case 'scrapeProp':
                    const propName = value || 'innerText';
                    const propVal = await loc.first().evaluate((el, p) => el[p], propName);
                    const propStoreKey = this.sanitize(step.inputKey || target);
                    const finalPropVal = propVal ? String(propVal).trim() : '';
                    context.store.set(propStoreKey, finalPropVal);
                    if (context.isBuffering) {
                        context.extractionBuffer[propStoreKey] = finalPropVal;
                    }
                    Logger.log(`Scraped property ${propName}: ${propVal} -> ${propStoreKey}`);
                    break;
                case 'scrapeItems':
                    if (!value)
                        throw new Error('scrapeItems requires a mapping JSON string in "value"');
                    const mapping = JSON.parse(value);
                    const rows = await loc.all();
                    const items = [];
                    for (const row of rows) {
                        if (context.isStopped)
                            throw new Error('STOP_SIGNAL: ScrapeItems halted');
                        const item = {};
                        for (const [key, innerSelector] of Object.entries(mapping)) {
                            const text = await row.locator(innerSelector).first().textContent();
                            const cleanKey = this.sanitize(key);
                            item[cleanKey] = this.sanitize(text);
                        }
                        items.push(item);
                        if (context.onItemExtracted) {
                            await context.onItemExtracted(item);
                        }
                        if (context.scenario.isPreview)
                            break; // Limit to 1 in preview
                    }
                    context.extractedItems.push(...items);
                    Logger.log(`Extracted ${items.length} structured items from ${target} ${context.scenario.isPreview ? '(Preview Limit: 1)' : ''}`);
                    break;
                case 'scrapeDeep':
                    const deepRows = await loc.all();
                    let subSteps = step.steps;
                    if ((!subSteps || subSteps.length === 0) && step.value) {
                        try {
                            // Backward compatibility: try to parse steps from value string
                            const parsed = JSON.parse(step.value);
                            if (Array.isArray(parsed)) {
                                subSteps = parsed;
                                Logger.log(`Recovered ${subSteps.length} sub-steps from legacy 'value' property for ${target}`);
                            }
                        }
                        catch (e) {
                            Logger.warn(`scrapeDeep on ${target} has 'value' but it's not valid JSON steps.`);
                        }
                    }
                    if (!subSteps || subSteps.length === 0) {
                        Logger.warn(`scrapeDeep on ${target} has no steps. Skipping.`);
                        break;
                    }
                    let pageNum = 1;
                    let hasMore = true;
                    while (hasMore) {
                        if (context.isStopped)
                            throw new Error('STOP_SIGNAL: ScrapeDeep loop halted');
                        // WAIT for the loop elements to be present
                        await loc.first().waitFor({ state: 'visible', timeout: 30000 }).catch(() => { });
                        // STABILIZATION: Give dynamic tables time to render all rows
                        await page.waitForTimeout(2000);
                        // Use direct page.$$ evaluation to ensure we get all matches regardless of locator state
                        const elements = await page.$$(selector);
                        const currentRows = elements;
                        // DIAGNOSTICS: Log text of all items found to see if they are valid
                        for (let idx = 0; idx < Math.min(elements.length, 10); idx++) {
                            const text = await elements[idx].innerText().catch(() => 'N/A');
                            Logger.log(`[Diagnostic] Match ${idx + 1}/${elements.length}: "${text.trim().substring(0, 50)}..."`);
                        }
                        Logger.log(`--------------------------------------------------`);
                        Logger.log(`📊 DEEP LOOP | PAGE ${pageNum} | FOUND ${currentRows.length} ITEMS`);
                        Logger.log(`--------------------------------------------------`);
                        // Capture screenshot and emit to UI for immediate verification
                        const screenshot = await page.screenshot({ fullPage: true, type: 'jpeg', quality: 60 }).catch(() => null);
                        if (screenshot) {
                            Logger.log(`[Diagnostic] Captured page screenshot for verification.`, {
                                screenshot: screenshot.toString('base64'),
                                pageNum
                            });
                        }
                        if (currentRows.length === 0) {
                            Logger.warn(`Deep Loop: No items found on page ${pageNum}.`);
                            break;
                        }
                        for (let i = 0; i < currentRows.length; i++) {
                            if (context.isStopped)
                                throw new Error('STOP_SIGNAL: ScrapeDeep iteration halted');
                            const iteration = (pageNum - 1) * currentRows.length + (i + 1);
                            const metadata = { loop_iteration: iteration };
                            // Re-fetch EVERYTHING after each navigation using direct evaluation
                            await loc.first().waitFor({ state: 'visible' }).catch(() => { });
                            const rows = await page.$$(selector);
                            const currentRow = rows[i];
                            if (!currentRow) {
                                Logger.warn(`Deep Loop: Item ${i} disappeared after navigation back. Skipping.`);
                                continue;
                            }
                            Logger.log(`Deep Loop [Iteration ${iteration}]: Item ${i + 1}/${rows.length} (Page ${pageNum})...`, metadata);
                            // Scroll and click
                            await currentRow.scrollIntoViewIfNeeded().catch(() => { });
                            await currentRow.click();
                            // Give SPA time to react to the click before executing sub-flow
                            await page.waitForTimeout(1000);
                            await page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => { });
                            await page.waitForLoadState('load');
                            // Execute sub-flow on the detail page with iteration context
                            // ENABLE BUFFERING: Collect all extractions from this sub-page into one object
                            context.isBuffering = true;
                            context.extractionBuffer = { ...metadata }; // Include iteration info
                            await DSLExecutor.executeFlow(context, subSteps, async (s, idx) => {
                                Logger.log(`[Iter ${iteration}] Step ${idx + 1}: ${s.action}`, metadata);
                            });
                            // FLUSH BUFFER: Save the combined results as a single document
                            if (Object.keys(context.extractionBuffer).length > 0) {
                                if (context.onItemExtracted)
                                    await context.onItemExtracted(context.extractionBuffer);
                                context.extractedItems.push(context.extractionBuffer);
                            }
                            context.isBuffering = false;
                            context.extractionBuffer = {};
                            // Return to list view
                            await page.goBack();
                            await page.waitForTimeout(1000);
                            await page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => { });
                            await page.waitForLoadState('load');
                            if (context.scenario.isPreview && i >= 2) {
                                Logger.log(`[Executor:Preview] Deep Loop truncated to 3 items for rapid verification.`, metadata);
                                break;
                            }
                        }
                        if (step.paginateTarget && !context.scenario.isPreview) {
                            const nextBtn = page.locator(step.paginateTarget).first();
                            const isVisible = await nextBtn.isVisible();
                            if (isVisible) {
                                Logger.log(`Deep Loop: Navigating to next page using ${step.paginateTarget}`);
                                await nextBtn.click();
                                await page.waitForLoadState('networkidle', { timeout: 30000 }).catch(() => { });
                                pageNum++;
                            }
                            else {
                                hasMore = false;
                            }
                        }
                        else {
                            hasMore = false;
                        }
                    }
                    Logger.log(`Deep Loop completed for ${target} after ${pageNum} pages.`);
                    break;
                case 'scrapePairs':
                    const pairContainer = loc.first();
                    const children = await pairContainer.locator('> *').all();
                    const pairItem = {};
                    Logger.log(`[Executor] Scraping pairs from ${children.length} candidate elements.`);
                    // Check if children are TRs (Table Mode) or flat interleaved (DT/DD or nested)
                    const firstTag = children.length > 0 ? await children[0].evaluate(el => el.tagName) : '';
                    if (firstTag === 'TR' || firstTag === 'TBODY') {
                        const rows = firstTag === 'TBODY' ? await pairContainer.locator('tr').all() : children;
                        for (const row of rows) {
                            const cells = await row.locator('td, th').all();
                            let cellIdx = 0;
                            while (cellIdx < cells.length - 1) {
                                const rawKey = await cells[cellIdx].textContent();
                                const key = this.sanitizeKey(rawKey);
                                if (!key) {
                                    cellIdx++;
                                    continue;
                                }
                                // Find the value: skip the key cell, and any cells that are just ":" or empty
                                let valText = "";
                                let foundValue = false;
                                let j = cellIdx + 1;
                                for (; j < cells.length; j++) {
                                    const text = await cells[j].textContent();
                                    const trimmed = text?.trim();
                                    if (trimmed && trimmed !== ':' && trimmed !== ':-') {
                                        valText = trimmed;
                                        foundValue = true;
                                        cellIdx = j + 1; // Move beyond the value cell
                                        break;
                                    }
                                }
                                if (foundValue) {
                                    pairItem[key] = valText;
                                    if (context.scenario.isPreview && Object.keys(pairItem).length >= 5)
                                        break; // Limit to 5 pairs in preview
                                }
                                else {
                                    cellIdx++; // No value found for this key, move to next cell
                                }
                            }
                        }
                    }
                    else {
                        // Flat mode (interleaved)
                        for (let i = 0; i < children.length - 1; i += 2) {
                            const rawKey = await children[i].textContent();
                            const key = this.sanitizeKey(rawKey);
                            const val = await children[i + 1].textContent();
                            if (key) {
                                pairItem[key] = this.sanitize(val);
                                if (context.scenario.isPreview && (i / 2) >= 4)
                                    break; // Limit to 5 pairs in preview
                            }
                        }
                    }
                    if (Object.keys(pairItem).length > 0) {
                        if (context.isBuffering) {
                            context.extractionBuffer = { ...context.extractionBuffer, ...pairItem };
                        }
                        else {
                            context.extractedItems.push(pairItem);
                            if (context.onItemExtracted)
                                await context.onItemExtracted(pairItem);
                        }
                    }
                    break;
                case 'storeResult':
                    const outputKey = step.mapTo || 'fileUrl';
                    const fileUrl = context.store.get('fileUrl');
                    if (fileUrl) {
                        context.store.set(outputKey, fileUrl);
                        Logger.log(`Result mapped: ${outputKey} -> ${fileUrl}`);
                    }
                    break;
                case 'eval':
                    if (!step.code && !value)
                        throw new Error(`Code required for 'eval' action on ${target}`);
                    const script = step.code || value;
                    // If the target looks like a placeholder or isn't found in registry, 
                    // we prefer page.evaluate to avoid waiting for a non-existent element.
                    const isPageLevel = target === 'page' || target === 'document' || target.includes('eval_step') || selector === target;
                    if (isPageLevel) {
                        Logger.log(`Executing page-level script...`);
                        const result = await page.evaluate((scriptBody) => {
                            try {
                                const fn = new Function('el', scriptBody);
                                return fn(null); // No element in page-level eval
                            }
                            catch (e) {
                                return { error: e.message };
                            }
                        }, script);
                        this.handleEvalResult(context, step, result);
                    }
                    else {
                        Logger.log(`Executing custom script on ${target}...`);
                        // We provide the element as 'el' and the current store as 'store'
                        const result = await locator.first().evaluate((el, scriptBody) => {
                            try {
                                const fn = new Function('el', scriptBody);
                                return fn(el);
                            }
                            catch (e) {
                                return { error: e.message };
                            }
                        }, script);
                        this.handleEvalResult(context, step, result);
                    }
                    break;
                case 'regex':
                    const pattern = step.regex || value;
                    if (!pattern)
                        throw new Error(`Regex pattern required for 'regex' action on ${target}`);
                    const rawText = await locator.first().textContent();
                    const matches = rawText?.match(new RegExp(pattern));
                    const regexResult = matches ? (matches[1] || matches[0]) : null;
                    const regexStoreKey = step.inputKey || step.label || target;
                    context.store.set(regexStoreKey, regexResult);
                    Logger.log(`Regex match for ${pattern}: ${regexResult} -> ${regexStoreKey}`);
                    break;
                case 'assert': {
                    const expected = value || step.value;
                    const actualText = await locator.first().textContent();
                    if (this.sanitize(actualText) !== this.sanitize(expected)) {
                        throw new Error(`Assertion failed on ${target}: Expected "${expected}", but found "${actualText}"`);
                    }
                    Logger.log(`Assertion passed on ${target}: found "${actualText}"`);
                    break;
                }
                case 'screenshot':
                    const screenshotDir = path.resolve(process.cwd(), 'output/screenshots');
                    if (!fs.existsSync(screenshotDir))
                        fs.mkdirSync(screenshotDir, { recursive: true });
                    const fileName = `${step.inputKey || target.split('.').pop()}_${Date.now()}.png`;
                    const screenshotPath = path.join(screenshotDir, fileName);
                    await locator.first().screenshot({ path: screenshotPath });
                    context.store.set(step.inputKey || 'last_screenshot', screenshotPath);
                    Logger.log(`Screenshot captured: ${screenshotPath}`);
                    break;
                default:
                    throw new Error(`Unsupported DSL action: ${action}`);
            }
        };
        try {
            await perform(locator.first());
        }
        catch (error) {
            Logger.warn(`Action failed on ${target}: ${error.message}. Attempting self-healing...`);
            // 1. Resolve all possible candidates from the centralized registry
            const candidates = SelectorResolver.resolveMany(target);
            const uniqueCandidates = [...new Set(candidates)];
            // 2. Trigger intelligent healing (Candidates -> LLM)
            const healedLocator = await Healer.heal(page, uniqueCandidates, this.aiConfig || undefined, { action, target, error: error.message });
            if (healedLocator) {
                Logger.log(`[RECOVERY] Healer successfully restored interaction. Retrying...`);
                await perform(healedLocator);
            }
            else {
                throw error;
            }
        }
    }
    static async applyStealth(page) {
        const ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36';
        try {
            // THE NUCLEAR OPTION: Use CDP to override User Agent and Client Hints at the browser engine level.
            // This is the only way to fix the "uafvl=HeadlessChrome" leak in Google Analytics/reCAPTCHA.
            const client = await page.context().newCDPSession(page);
            await client.send('Emulation.setUserAgentOverride', {
                userAgent: ua,
                platform: 'Windows',
                userAgentMetadata: {
                    brands: [
                        { brand: 'Not(A:Brand', version: '8' },
                        { brand: 'Google Chrome', version: '123' },
                        { brand: 'Chromium', version: '123' }
                    ],
                    fullVersionList: [
                        { brand: 'Not(A:Brand', version: '8.0.0.0' },
                        { brand: 'Google Chrome', version: '123.0.6312.122' },
                        { brand: 'Chromium', version: '123.0.6312.122' }
                    ],
                    platform: 'Windows',
                    platformVersion: '10.0.0',
                    architecture: 'x86',
                    model: '',
                    mobile: false,
                    bitness: '64'
                }
            });
        }
        catch (e) {
            Logger.warn('CDP Stealth failed, falling back to header overrides');
            await page.setExtraHTTPHeaders({
                'User-Agent': ua,
                'Sec-CH-UA': '"Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"',
                'Sec-CH-UA-Mobile': '?0',
                'Sec-CH-UA-Platform': '"Windows"'
            });
        }
        // 2. Browser Level: Mask JavaScript signatures
        // Use string-based injection to avoid bundler transformations (like __name injection)
        await page.addInitScript(`
      (function(userAgent) {
        // Spoof navigator properties
        Object.defineProperty(navigator, 'userAgent', { get: () => userAgent });
        Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
        Object.defineProperty(navigator, 'webdriver', { get: () => false });
        
        // Spoof Modern Client Hints
        try {
          const mockUAD = {
            brands: [
              { brand: 'Not(A:Brand', version: '8' },
              { brand: 'Google Chrome', version: '123' },
              { brand: 'Chromium', version: '123' }
            ],
            mobile: false,
            platform: 'Windows',
            getHighEntropyValues: () => Promise.resolve({
              architecture: 'x86',
              bitness: '64',
              model: '',
              platformVersion: '10.0.0',
              fullVersionList: [
                 { brand: 'Not(A:Brand', version: '8.0.0.0' },
                 { brand: 'Google Chrome', version: '123.0.6312.122' },
                 { brand: 'Chromium', version: '123.0.6312.122' }
              ]
            })
          };
          Object.defineProperty(navigator, 'userAgentData', { get: () => mockUAD });
        } catch (e) {}
        
        navigator.languages = ['en-US', 'en'];

        // 3. Hardware Fingerprinting Masks
        try {
          Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
          Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 4 });
          Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 });
        } catch (e) {}

        // 4. WebGL Masking
        try {
          const getParameter = WebGLRenderingContext.prototype.getParameter;
          WebGLRenderingContext.prototype.getParameter = function(parameter) {
            // UNMASKED_VENDOR_WEBGL
            if (parameter === 37445) return 'Google Inc. (NVIDIA)';
            // UNMASKED_RENDERER_WEBGL
            if (parameter === 37446) return 'ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0, D3D11)';
            return getParameter.apply(this, arguments);
          };
        } catch (e) {}

        // 5. Plugin & MimeType Masks
        try {
          Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });
          Object.defineProperty(navigator, 'mimeTypes', { get: () => [1, 2, 3, 4, 5] });
        } catch (e) {}
        // 6. Canvas Fingerprinting (Subtle jitter)
        const toDataURL = HTMLCanvasElement.prototype.toDataURL;
        HTMLCanvasElement.prototype.toDataURL = function(type, quality) {
          // We could add noise here, but for now we just ensure it is not a headless leak
          return toDataURL.call(this, type, quality);
        };

          
        Object.defineProperty(window, 'screen', {
            get: () => ({
              width: 1920,
              height: 1080,
              availWidth: 1920,
              availHeight: 1040,
              colorDepth: 24,
              pixelDepth: 24
            })
          });

          // Mask Notification permission
          const originalQuery = window.navigator.permissions.query;
          window.navigator.permissions.query = (parameters) => (
            parameters.name === 'notifications' ?
              Promise.resolve({ state: Notification.permission }) :
              originalQuery(parameters)
          );

          // Mask AudioContext
          const originalAudioContext = window.AudioContext || window.webkitAudioContext;
          if (originalAudioContext) {
            window.AudioContext = window.webkitAudioContext = class extends originalAudioContext {
              constructor() {
                super();
              }
            };
          }

          // Mask Chrome Runtime
          window.chrome = {
            app: {
              isInstalled: false,
              InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
              RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' }
            },
            runtime: {
              OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' },
              OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' },
              PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86_32', X86_64: 'x86_64' },
              PlatformNaclArch: { ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86_32', X86_64: 'x86_64' },
              PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' },
              RequestUpdateCheckStatus: { NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available' }
            }
          };

          // Mask Hardware Concurrency
          Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
          Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });

        } catch (e) {}
      })("${ua}");
    `);
        // 4. CSP Bypass: Strip Content-Security-Policy headers
        await page.route('**/*', async (route) => {
            try {
                const response = await route.fetch();
                const headers = { ...response.headers() };
                delete headers['content-security-policy'];
                delete headers['content-security-policy-report-only'];
                await route.fulfill({ response, headers });
            }
            catch (e) {
                await route.continue().catch(() => { });
            }
        });
    }
    /**
     * Processes the result of a JS evaluation.
     */
    static handleEvalResult(context, step, result) {
        if (result && typeof result === 'object' && !Array.isArray(result)) {
            if (result.error)
                throw new Error(`JS Eval Error: ${result.error}`);
            // Merge returned object into store
            for (const [k, v] of Object.entries(result)) {
                context.store.set(k, v);
            }
            Logger.log(`Eval result merged into store: ${Object.keys(result).join(', ')}`);
        }
        else if (step.inputKey) {
            context.store.set(step.inputKey, result);
            Logger.log(`Eval result stored as ${step.inputKey}: ${result}`);
        }
    }
}
