"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getPluginSeverity = getPluginSeverity;
exports.resolvePluginConfig = resolvePluginConfig;
exports.getTestCount = getTestCount;
exports.calculateTotalTests = calculateTotalTests;
exports.getMultilingualRequestedCount = getMultilingualRequestedCount;
exports.synthesize = synthesize;
const fs = __importStar(require("fs"));
const async_1 = __importDefault(require("async"));
const chalk_1 = __importDefault(require("chalk"));
const cli_progress_1 = __importDefault(require("cli-progress"));
const cli_table3_1 = __importDefault(require("cli-table3"));
const js_yaml_1 = __importDefault(require("js-yaml"));
const cliState_1 = __importDefault(require("../cliState"));
const envars_1 = require("../envars");
const logger_1 = __importStar(require("../logger"));
const apiHealth_1 = require("../util/apiHealth");
const invariant_1 = __importDefault(require("../util/invariant"));
const templates_1 = require("../util/templates");
const constants_1 = require("./constants");
const entities_1 = require("./extraction/entities");
const purpose_1 = require("./extraction/purpose");
const custom_1 = require("./plugins/custom");
const index_1 = require("./plugins/index");
const shared_1 = require("./providers/shared");
const remoteGeneration_1 = require("./remoteGeneration");
const index_2 = require("./strategies/index");
const util_1 = require("./strategies/util");
const util_2 = require("./util");
const multilingual_1 = require("./strategies/multilingual");
function getPolicyText(metadata) {
    if (!metadata || metadata.policy === undefined || metadata.policy === null) {
        return undefined;
    }
    const policyValue = metadata.policy;
    if (typeof policyValue === 'string') {
        return policyValue;
    }
    if (typeof policyValue === 'object') {
        const policyObject = policyValue;
        return typeof policyObject.text === 'string' && policyObject.text.length > 0
            ? policyObject.text
            : undefined;
    }
    return undefined;
}
const MAX_MAX_CONCURRENCY = 20;
/**
 * Gets the severity level for a plugin based on its ID and configuration.
 * @param pluginId - The ID of the plugin.
 * @param pluginConfig - Optional configuration for the plugin.
 * @returns The severity level.
 */
function getPluginSeverity(pluginId, pluginConfig) {
    if (pluginConfig?.severity) {
        return pluginConfig.severity;
    }
    const shortId = (0, util_2.getShortPluginId)(pluginId);
    return shortId in constants_1.riskCategorySeverityMap
        ? constants_1.riskCategorySeverityMap[shortId]
        : constants_1.Severity.Low;
}
/**
 * Determines the status of test generation based on requested and generated counts.
 * @param requested - The number of requested tests.
 * @param generated - The number of generated tests.
 * @returns A colored string indicating the status.
 */
function getStatus(requested, generated) {
    if (requested === 0 && generated === 0) {
        return chalk_1.default.gray('Skipped');
    }
    if (generated === 0) {
        return chalk_1.default.red('Failed');
    }
    if (generated < requested) {
        return chalk_1.default.yellow('Partial');
    }
    return chalk_1.default.green('Success');
}
/**
 * Generates a report of plugin and strategy results.
 * @param pluginResults - Results from plugin executions.
 * @param strategyResults - Results from strategy executions.
 * @returns A formatted string containing the report.
 */
function generateReport(pluginResults, strategyResults) {
    const table = new cli_table3_1.default({
        head: ['#', 'Type', 'ID', 'Requested', 'Generated', 'Status'].map((h) => chalk_1.default.dim(chalk_1.default.white(h))),
        colWidths: [5, 10, 40, 12, 12, 14],
    });
    let rowIndex = 1;
    Object.entries(pluginResults)
        .sort((a, b) => a[0].localeCompare(b[0]))
        .forEach(([id, { requested, generated }]) => {
        table.push([rowIndex++, 'Plugin', id, requested, generated, getStatus(requested, generated)]);
    });
    Object.entries(strategyResults)
        .sort((a, b) => a[0].localeCompare(b[0]))
        .forEach(([id, { requested, generated }]) => {
        table.push([
            rowIndex++,
            'Strategy',
            id,
            requested,
            generated,
            getStatus(requested, generated),
        ]);
    });
    return `\nTest Generation Report:\n${table.toString()}`;
}
/**
 * Resolves top-level file paths in the plugin configuration.
 * @param config - The plugin configuration to resolve.
 * @returns The resolved plugin configuration.
 */
function resolvePluginConfig(config) {
    if (!config) {
        return {};
    }
    for (const key in config) {
        const value = config[key];
        if (typeof value === 'string' && value.startsWith('file://')) {
            const filePath = value.slice('file://'.length);
            if (!fs.existsSync(filePath)) {
                throw new Error(`File not found: ${filePath}`);
            }
            if (filePath.endsWith('.yaml')) {
                config[key] = js_yaml_1.default.load(fs.readFileSync(filePath, 'utf8'));
            }
            else if (filePath.endsWith('.json')) {
                config[key] = JSON.parse(fs.readFileSync(filePath, 'utf8'));
            }
            else {
                config[key] = fs.readFileSync(filePath, 'utf8');
            }
        }
    }
    return config;
}
const categories = {
    foundation: constants_1.FOUNDATION_PLUGINS,
    harmful: Object.keys(constants_1.HARM_PLUGINS),
    bias: constants_1.BIAS_PLUGINS,
    pii: constants_1.PII_PLUGINS,
    medical: constants_1.MEDICAL_PLUGINS,
    pharmacy: constants_1.PHARMACY_PLUGINS,
    insurance: constants_1.INSURANCE_PLUGINS,
    financial: constants_1.FINANCIAL_PLUGINS,
};
/**
 * Formats the test count for display.
 * @param numTests - The number of tests.
 * @param strategy - Whether the test count is for a strategy.
 * @returns A formatted string representing the test count.
 */
const formatTestCount = (numTests, strategy) => numTests === 1
    ? `1 ${strategy ? 'additional ' : ''}test`
    : `${numTests} ${strategy ? 'additional ' : ''}tests`;
/**
 * Gets the language from a test case's metadata.
 * Checks both metadata.language and metadata.modifiers.language.
 * @param test - The test case to get language from.
 * @returns The language string or undefined if not found.
 */
function getLanguageForTestCase(test) {
    if (!test) {
        return undefined;
    }
    return test.metadata?.language || test.metadata?.modifiers?.language;
}
/**
 * Adds comprehensive metadata to plugin test cases including language, plugin info, and severity.
 * @param test - The test case to add metadata to.
 * @param lang - The language for this test.
 * @param plugin - The plugin configuration.
 * @param testGenerationInstructions - Optional test generation instructions.
 * @returns Test case with complete metadata.
 */
function addLanguageToPluginMetadata(test, lang, plugin, testGenerationInstructions) {
    const existingLanguage = getLanguageForTestCase(test);
    const languageToAdd = lang && !existingLanguage ? { language: lang } : {};
    return {
        ...test,
        metadata: {
            pluginId: plugin.id,
            pluginConfig: resolvePluginConfig(plugin.config),
            severity: plugin.severity ?? getPluginSeverity(plugin.id, resolvePluginConfig(plugin.config)),
            modifiers: {
                ...(testGenerationInstructions ? { testGenerationInstructions } : {}),
                ...(plugin.config?.modifiers || {}),
                ...test.metadata?.modifiers,
                // Add language to modifiers if not already present (respect existing)
                ...languageToAdd,
            },
            ...test.metadata,
            // Hoist language to top-level metadata for backward compatibility and easier access
            ...languageToAdd,
        },
    };
}
/**
 * Determines whether a strategy should be applied to a test case based on plugin targeting rules.
 *
 * This function evaluates multiple criteria to decide if a strategy matches a test case:
 * - Excludes strategy-exempt plugins (defined in STRATEGY_EXEMPT_PLUGINS)
 * - Excludes sequence providers (which are verbatim and don't support strategies)
 * - Respects plugin-level strategy exclusions via excludeStrategies config
 * - Matches against target plugins through direct ID match or category prefixes
 *
 * @param testCase - The test case containing plugin metadata to evaluate
 * @param strategyId - The ID of the strategy being considered for application
 * @param targetPlugins - Optional array of plugin IDs or categories that the strategy targets.
 *                       If undefined or empty, strategy applies to all non-exempt plugins.
 *                       Supports both exact matches and category prefixes (e.g., 'harmful' matches 'harmful:hate')
 * @returns True if the strategy should be applied to this test case, false otherwise
 */
/**
 * Applies strategies to the test cases.
 * @param testCases - The initial test cases generated by plugins.
 * @param strategies - The strategies to apply.
 * @param injectVar - The variable to inject.
 * @returns An array of new test cases generated by strategies.
 */
async function applyStrategies(testCases, strategies, injectVar, excludeTargetOutputFromAgenticAttackGeneration) {
    const newTestCases = [];
    const strategyResults = {};
    for (const strategy of strategies) {
        logger_1.default.debug(`Generating ${strategy.id} tests`);
        let strategyAction;
        if (strategy.id.startsWith('file://')) {
            const loadedStrategy = await (0, index_2.loadStrategy)(strategy.id);
            strategyAction = loadedStrategy.action;
        }
        else {
            // First try to find the exact strategy ID (e.g., jailbreak:composite)
            let builtinStrategy = index_2.Strategies.find((s) => s.id === strategy.id);
            // If not found, handle custom strategy variants (e.g., custom:aggressive)
            if (!builtinStrategy && strategy.id.includes(':')) {
                const baseStrategyId = strategy.id.split(':')[0];
                builtinStrategy = index_2.Strategies.find((s) => s.id === baseStrategyId);
            }
            if (!builtinStrategy) {
                logger_1.default.warn(`Strategy ${strategy.id} not registered, skipping`);
                continue;
            }
            strategyAction = builtinStrategy.action;
        }
        const targetPlugins = strategy.config?.plugins;
        const applicableTestCases = testCases.filter((t) => (0, util_1.pluginMatchesStrategyTargets)(t, strategy.id, targetPlugins));
        const strategyTestCases = await strategyAction(applicableTestCases, injectVar, {
            ...(strategy.config || {}),
            excludeTargetOutputFromAgenticAttackGeneration,
        }, strategy.id);
        newTestCases.push(...strategyTestCases
            .filter((t) => t !== null && t !== undefined)
            .map((t) => ({
            ...t,
            metadata: {
                ...(t?.metadata || {}),
                strategyId: t?.metadata?.strategyId || strategy.id,
                ...(t?.metadata?.pluginId && { pluginId: t.metadata.pluginId }),
                ...(t?.metadata?.pluginConfig && {
                    pluginConfig: t.metadata.pluginConfig,
                }),
                ...(strategy.config && {
                    strategyConfig: {
                        ...strategy.config,
                        ...(t?.metadata?.strategyConfig || {}),
                    },
                }),
            },
        })));
        // Compute a display id for reporting (helpful for layered strategies)
        const displayId = strategy.id === 'layer' && Array.isArray(strategy.config?.steps)
            ? `layer(${strategy.config.steps
                .map((st) => (typeof st === 'string' ? st : st.id))
                .join('→')})`
            : strategy.id;
        // Check if strategy generated tests across multiple languages
        // Language can be in metadata.language or metadata.modifiers.language
        // Note: Language is passed to strategies via test metadata.modifiers.language from plugin generation
        const languagesInResults = new Set(strategyTestCases
            .map((t) => getLanguageForTestCase(t))
            .filter((lang) => lang !== undefined));
        if (languagesInResults.size > 1) {
            // Strategy was applied to multilingual tests - break down by language
            // Don't show language suffix for English (en) - it's the default
            const resultsByLanguage = {};
            for (const lang of languagesInResults) {
                const testsForLang = strategyTestCases.filter((t) => getLanguageForTestCase(t) === lang);
                const applicableForLang = applicableTestCases.filter((t) => getLanguageForTestCase(t) === lang);
                let n = 1;
                if (typeof strategy.config?.n === 'number') {
                    n = strategy.config.n;
                }
                else if ((0, constants_1.isFanoutStrategy)(strategy.id)) {
                    n = (0, constants_1.getDefaultNFanout)(strategy.id);
                }
                resultsByLanguage[lang] = {
                    requested: applicableForLang.length * n,
                    generated: testsForLang.length,
                };
            }
            for (const [lang, result] of Object.entries(resultsByLanguage)) {
                const strategyDisplayId = lang === 'en' ? displayId : `${displayId} (${lang})`;
                strategyResults[strategyDisplayId] = result;
            }
        }
        else if (strategy.id === 'layer') {
            // Layer strategy: requested count is same as applicable test cases
            strategyResults[displayId] = {
                requested: applicableTestCases.length,
                generated: strategyTestCases.length,
            };
        }
        else {
            // get an accurate 'Requested' count for strategies that add additional tests during generation
            let n = 1;
            if (typeof strategy.config?.n === 'number') {
                n = strategy.config.n;
            }
            else if ((0, constants_1.isFanoutStrategy)(strategy.id)) {
                n = (0, constants_1.getDefaultNFanout)(strategy.id);
            }
            strategyResults[displayId] = {
                requested: applicableTestCases.length * n,
                generated: strategyTestCases.length,
            };
        }
    }
    return { testCases: newTestCases, strategyResults };
}
/**
 * Helper function to get the test count based on strategy configuration.
 * @param strategy - The strategy object to evaluate.
 * @param totalPluginTests - The total number of plugin tests.
 * @param strategies - The array of strategies.
 * @returns The calculated test count.
 */
function getTestCount(strategy, totalPluginTests, _strategies) {
    // Basic strategy either keeps original count or removes all tests
    if (strategy.id === 'basic') {
        return strategy.config?.enabled === false ? 0 : totalPluginTests;
    }
    // Layer strategy: returns the same count as plugin tests
    // (plugins already account for language multipliers)
    if (strategy.id === 'layer') {
        return totalPluginTests;
    }
    // Retry strategy doubles the plugin tests
    if (strategy.id === 'retry') {
        const configuredNumTests = strategy.config?.numTests;
        const additionalTests = configuredNumTests ?? totalPluginTests;
        return totalPluginTests + additionalTests;
    }
    // Apply fan-out multiplier if this is a fan-out strategy
    let n = 1;
    if (typeof strategy.config?.n === 'number') {
        n = strategy.config.n;
    }
    else if ((0, constants_1.isFanoutStrategy)(strategy.id)) {
        n = (0, constants_1.getDefaultNFanout)(strategy.id);
    }
    return totalPluginTests * n;
}
/**
 * Calculates the total number of tests to be generated based on plugins and strategies.
 * @param plugins - The array of plugins to generate tests for
 * @param strategies - The array of strategies to apply
 * @returns Object containing total tests and intermediate calculations
 */
function calculateTotalTests(plugins, strategies, language) {
    const retryStrategy = strategies.find((s) => s.id === 'retry');
    const basicStrategy = strategies.find((s) => s.id === 'basic');
    const audioStrategy = strategies.find((s) => s.id === 'audio');
    const basicStrategyExists = basicStrategy !== undefined;
    const includeBasicTests = audioStrategy ? false : basicStrategy?.config?.enabled ?? true;
    const effectiveStrategyCount = basicStrategyExists && !includeBasicTests ? strategies.length - 1 : strategies.length;
    // Calculate total plugin tests accounting for multiple languages
    // Each plugin's tests are multiplied by the number of languages when multiple languages are configured
    const totalPluginTests = plugins.reduce((sum, p) => {
        const pluginLanguageConfig = p.config?.language ?? language;
        const pluginLanguageCount = Array.isArray(pluginLanguageConfig)
            ? pluginLanguageConfig.length
            : 1;
        return sum + (p.numTests || 0) * pluginLanguageCount;
    }, 0);
    // When there are no strategies, or only a disabled basic strategy
    if (strategies.length === 0 ||
        (strategies.length === 1 && basicStrategyExists && !includeBasicTests)) {
        return {
            effectiveStrategyCount: 0,
            includeBasicTests: strategies.length === 0 ? true : includeBasicTests,
            totalPluginTests,
            totalTests: includeBasicTests ? totalPluginTests : 0,
        };
    }
    // Start with base test count from basic strategy
    let totalTests = includeBasicTests ? totalPluginTests : 0;
    // Apply retry strategy first if present
    if (retryStrategy) {
        totalTests = getTestCount(retryStrategy, totalTests, strategies);
    }
    // Apply other non-basic, non-retry strategies
    for (const strategy of strategies) {
        if (['basic', 'retry'].includes(strategy.id)) {
            continue;
        }
        // Add the tests from this strategy to the total, not replace the total
        totalTests += getTestCount(strategy, totalPluginTests, strategies);
    }
    return {
        effectiveStrategyCount,
        includeBasicTests,
        totalPluginTests,
        totalTests,
    };
}
/**
 * Type guard to check if a strategy ID is a strategy collection
 */
function isStrategyCollection(id) {
    return constants_1.STRATEGY_COLLECTIONS.includes(id);
}
function getMultilingualRequestedCount(testCases, strategy) {
    // If languages is an empty array, return 0
    if (Array.isArray(strategy.config?.languages) && strategy.config?.languages.length === 0) {
        return 0;
    }
    const numLanguages = Array.isArray(strategy.config?.languages)
        ? strategy.config?.languages.length ?? 0
        : multilingual_1.DEFAULT_LANGUAGES.length;
    return testCases.length * numLanguages;
}
/**
 * Synthesizes test cases based on provided options.
 * @param options - The options for test case synthesis.
 * @returns A promise that resolves to an object containing the purpose, entities, and test cases.
 */
async function synthesize({ abortSignal, delay, entities: entitiesOverride, injectVar, language, maxConcurrency = 1, plugins, prompts, provider, purpose: purposeOverride, strategies, targetLabels, showProgressBar: showProgressBarOverride, excludeTargetOutputFromAgenticAttackGeneration, testGenerationInstructions, workspaceId, }) {
    // Add abort check helper
    const checkAbort = () => {
        if (abortSignal?.aborted) {
            throw new Error('Operation cancelled');
        }
    };
    // Add abort checks at key points
    checkAbort();
    if (prompts.length === 0) {
        throw new Error('Prompts array cannot be empty');
    }
    if (delay && maxConcurrency > 1) {
        maxConcurrency = 1;
        logger_1.default.warn('Delay is enabled, setting max concurrency to 1.');
    }
    if (maxConcurrency > MAX_MAX_CONCURRENCY) {
        maxConcurrency = MAX_MAX_CONCURRENCY;
        logger_1.default.info(`Max concurrency for test generation is capped at ${MAX_MAX_CONCURRENCY}.`);
    }
    const expandedStrategies = [];
    strategies.forEach((strategy) => {
        if (isStrategyCollection(strategy.id)) {
            const aliasedStrategies = constants_1.STRATEGY_COLLECTION_MAPPINGS[strategy.id];
            if (aliasedStrategies) {
                aliasedStrategies.forEach((strategyId) => {
                    expandedStrategies.push({
                        id: strategyId,
                        config: {
                            ...strategy.config,
                            workspaceId,
                        },
                    });
                });
            }
            else {
                logger_1.default.warn(`Strategy collection ${strategy.id} has no mappings, skipping`);
            }
        }
        else {
            expandedStrategies.push({
                ...strategy,
                config: {
                    ...strategy.config,
                    workspaceId,
                },
            });
        }
    });
    // Deduplicate strategies by a key. For most strategies, the key is the id.
    // For 'layer', use label if provided, otherwise include the ordered step ids.
    const seen = new Set();
    const keyForStrategy = (s) => {
        if (s.id === 'layer' && s.config) {
            const config = s.config;
            // If label is provided, use it for uniqueness (allows multiple layer strategies)
            if (typeof config.label === 'string' && config.label.trim()) {
                return `layer/${config.label}`;
            }
            // Otherwise use steps for uniqueness
            if (Array.isArray(config.steps)) {
                const steps = config.steps.map((st) => typeof st === 'string' ? st : (st?.id ?? 'unknown'));
                return `layer:${steps.join('->')}`;
            }
        }
        return s.id;
    };
    strategies = expandedStrategies.filter((strategy) => {
        const key = keyForStrategy(strategy);
        if (seen.has(key)) {
            logger_1.default.debug(`[Synthesize] Skipping duplicate strategy: ${key}`);
            return false;
        }
        seen.add(key);
        return true;
    });
    (0, index_2.validateStrategies)(strategies);
    // If any language-disallowed strategies are present, force language to English only
    const hasLanguageDisallowedStrategy = strategies.some((s) => (0, constants_1.isLanguageDisallowedStrategy)(s.id));
    if (hasLanguageDisallowedStrategy && language) {
        const originalLanguage = Array.isArray(language) ? language.join(', ') : language;
        language = 'en';
        logger_1.default.info(`[Language Override] Detected language-disallowed strategy (audio/video/image/layer/math-prompt). Forcing language to 'en' (was: ${originalLanguage})`);
    }
    const redteamProvider = await shared_1.redteamProviderManager.getProvider({
        provider,
    });
    const { effectiveStrategyCount, includeBasicTests, totalPluginTests, totalTests } = calculateTotalTests(plugins, strategies, language);
    logger_1.default.info(`Synthesizing test cases for ${prompts.length} ${prompts.length === 1 ? 'prompt' : 'prompts'}...\nUsing plugins:\n\n${chalk_1.default.yellow(plugins
        .map((p) => {
        // Calculate actual test count accounting for multiple languages
        const pluginLanguageConfig = p.config?.language ?? language;
        const pluginLanguageCount = Array.isArray(pluginLanguageConfig)
            ? pluginLanguageConfig.length
            : 1;
        const actualTestCount = (p.numTests || 0) * pluginLanguageCount;
        return `${p.id} (${formatTestCount(actualTestCount, false)})${p.config ? ` (${JSON.stringify(p.config)})` : ''}`;
    })
        .sort()
        .join('\n'))}\n`);
    if (strategies.length > 0) {
        logger_1.default.info(`Using strategies:\n\n${chalk_1.default.yellow(strategies
            .filter((s) => !['basic', 'retry'].includes(s.id))
            .map((s) => {
            // For non-basic strategies, we want to show the additional tests they generate
            let testCount = totalPluginTests;
            // Apply fan-out multiplier if this is a fan-out strategy
            let n = 1;
            if (typeof s.config?.n === 'number') {
                n = s.config.n;
            }
            else if ((0, constants_1.isFanoutStrategy)(s.id)) {
                n = (0, constants_1.getDefaultNFanout)(s.id);
            }
            testCount = totalPluginTests * n;
            return `${s.id} (${formatTestCount(testCount, true)})`;
        })
            .sort()
            .join('\n'))}\n`);
    }
    logger_1.default.info(chalk_1.default.bold(`Test Generation Summary:`) +
        `\n• Total tests: ${chalk_1.default.cyan(totalTests)}` +
        `\n• Plugin tests: ${chalk_1.default.cyan(totalPluginTests)}` +
        `\n• Plugins: ${chalk_1.default.cyan(plugins.length)}` +
        `\n• Strategies: ${chalk_1.default.cyan(effectiveStrategyCount)}` +
        `\n• Max concurrency: ${chalk_1.default.cyan(maxConcurrency)}\n` +
        (delay ? `• Delay: ${chalk_1.default.cyan(delay)}\n` : ''));
    if (typeof injectVar !== 'string') {
        const parsedVars = (0, templates_1.extractVariablesFromTemplates)(prompts);
        if (parsedVars.length > 1) {
            logger_1.default.warn(`\nMultiple variables found in prompts: ${parsedVars.join(', ')}. Using the last one "${parsedVars[parsedVars.length - 1]}". Override this selection with --injectVar`);
        }
        else if (parsedVars.length === 0) {
            logger_1.default.warn('No variables found in prompts. Using "query" as the inject variable.');
        }
        // Odds are that the last variable is the user input since the user input usually goes at the end of the prompt
        injectVar = parsedVars[parsedVars.length - 1] || 'query';
        (0, invariant_1.default)(typeof injectVar === 'string', `Inject var must be a string, got ${injectVar}`);
    }
    // Expand plugins first
    for (const [category, categoryPlugins] of Object.entries(categories)) {
        const plugin = plugins.find((p) => p.id === category);
        if (plugin) {
            plugins.push(...categoryPlugins.map((p) => ({ id: p, numTests: plugin.numTests })));
        }
    }
    const expandedPlugins = [];
    const expandPlugin = (plugin, mapping) => {
        mapping.plugins.forEach((p) => expandedPlugins.push({ id: p, numTests: plugin.numTests }));
        strategies.push(...mapping.strategies.map((s) => ({ id: s })));
    };
    plugins.forEach((plugin) => {
        // First check if this is a direct plugin that should not be expanded
        // This is for plugins like bias:gender that have a prefix matching an alias
        const isDirectPlugin = index_1.Plugins.some((p) => p.key === plugin.id);
        if (isDirectPlugin) {
            expandedPlugins.push(plugin);
            return;
        }
        const mappingKey = Object.keys(constants_1.ALIASED_PLUGIN_MAPPINGS).find((key) => plugin.id === key || plugin.id.startsWith(`${key}:`));
        if (mappingKey) {
            const mapping = constants_1.ALIASED_PLUGIN_MAPPINGS[mappingKey][plugin.id] ||
                Object.values(constants_1.ALIASED_PLUGIN_MAPPINGS[mappingKey]).find((_m) => plugin.id.startsWith(`${mappingKey}:`));
            if (mapping) {
                expandPlugin(plugin, mapping);
            }
        }
        else {
            expandedPlugins.push(plugin);
        }
    });
    const validatePlugin = (plugin) => {
        if (Object.keys(categories).includes(plugin.id)) {
            return false;
        }
        const registeredPlugin = index_1.Plugins.find((p) => p.key === plugin.id);
        if (!registeredPlugin) {
            if (!plugin.id.startsWith('file://')) {
                logger_1.default.debug(`Plugin ${plugin.id} not registered, skipping validation`);
            }
        }
        else if (registeredPlugin.validate) {
            try {
                registeredPlugin.validate({
                    language,
                    modifiers: {
                        ...(testGenerationInstructions ? { testGenerationInstructions } : {}),
                        ...(plugin.config?.modifiers || {}),
                    },
                    ...resolvePluginConfig(plugin.config),
                });
            }
            catch (error) {
                logger_1.default.warn(`Validation failed for plugin ${plugin.id}: ${error}, skipping plugin.`);
                return false;
            }
        }
        return true;
    };
    // Validate all plugins upfront
    logger_1.default.debug('Validating plugins...');
    plugins = [...new Set(expandedPlugins)].filter(validatePlugin).sort();
    // Check API health before proceeding
    if ((0, remoteGeneration_1.shouldGenerateRemote)()) {
        const healthUrl = (0, remoteGeneration_1.getRemoteHealthUrl)();
        if (healthUrl) {
            logger_1.default.debug(`Checking Trusys API health at ${healthUrl}...`);
            const healthResult = await (0, apiHealth_1.checkRemoteHealth)(healthUrl);
            if (healthResult.status !== 'OK') {
                throw new Error(`Unable to proceed with test generation: ${healthResult.message}\n` +
                    'Please check your API configuration or try again later.');
            }
            logger_1.default.debug('API health check passed');
        }
    }
    // Start the progress bar
    let progressBar = null;
    const isWebUI = Boolean(cliState_1.default.webUI);
    const showProgressBar = !isWebUI &&
        (0, envars_1.getEnvString)('LOG_LEVEL') !== 'debug' &&
        (0, logger_1.getLogLevel)() !== 'debug' &&
        showProgressBarOverride !== false;
    if (showProgressBar) {
        progressBar = new cli_progress_1.default.SingleBar({
            format: 'Generating | {bar} | {percentage}% | {value}/{total} | {task}',
            gracefulExit: true,
        }, cli_progress_1.default.Presets.shades_classic);
        progressBar.start(totalPluginTests + 2, 0, { task: 'Initializing' });
    }
    // Replace progress bar updates with logger calls when in web UI
    if (showProgressBar) {
        progressBar?.increment(1, { task: 'Extracting system purpose' });
    }
    else {
        logger_1.default.info('Extracting system purpose...');
    }
    const purpose = purposeOverride || (await (0, purpose_1.extractSystemPurpose)(redteamProvider, prompts));
    if (showProgressBar) {
        progressBar?.increment(1, { task: 'Extracting entities' });
    }
    else {
        logger_1.default.info('Extracting entities...');
    }
    const entities = Array.isArray(entitiesOverride)
        ? entitiesOverride
        : await (0, entities_1.extractEntities)(redteamProvider, prompts);
    logger_1.default.debug(`System purpose: ${purpose}`);
    const pluginResults = {};
    const testCases = [];
    await async_1.default.forEachLimit(plugins, maxConcurrency, async (plugin) => {
        // Check for abort signal before generating tests
        checkAbort();
        if (showProgressBar) {
            progressBar?.update({ task: plugin.id });
        }
        else {
            logger_1.default.info(`Generating tests for ${plugin.id}...`);
        }
        const { action } = index_1.Plugins.find((p) => p.key === plugin.id) || {};
        if (action) {
            logger_1.default.debug(`Generating tests for ${plugin.id}...`);
            // If plugin has its own language, use that; otherwise use global language
            const languageConfig = plugin.config?.language ?? language;
            const languages = Array.isArray(languageConfig)
                ? languageConfig
                : languageConfig
                    ? [languageConfig]
                    : [undefined];
            logger_1.default.debug(`[Language Processing] Plugin: ${plugin.id}, Languages: ${JSON.stringify(languages)}, NumTests per language: ${plugin.numTests}${plugin.config?.language ? ' (plugin override)' : ''}`);
            // Generate tests for each language in parallel using Promise.allSettled
            const allPluginTests = [];
            const resultsPerLanguage = {};
            const languagePromises = languages.map(async (lang) => {
                const pluginTests = await action({
                    provider: redteamProvider,
                    purpose,
                    injectVar: injectVar || '',
                    n: plugin.numTests,
                    delayMs: delay || 0,
                    config: {
                        ...resolvePluginConfig(plugin.config),
                        ...(lang ? { language: lang } : {}),
                        modifiers: {
                            ...(testGenerationInstructions ? { testGenerationInstructions } : {}),
                            ...(plugin.config?.modifiers || {}),
                        },
                    },
                });
                {
                    const langKey = lang;
                    if (Array.isArray(pluginTests) && pluginTests.length > 0) {
                        // Add all metadata (language + plugin info) so strategies can access it
                        const testsWithMetadata = pluginTests.map((test) => addLanguageToPluginMetadata(test, lang, plugin, testGenerationInstructions));
                        return {
                            lang: langKey,
                            tests: testsWithMetadata,
                            requested: plugin.numTests,
                            generated: pluginTests.length,
                        };
                    }
                    logger_1.default.warn(`[Language Processing] No tests generated for ${plugin.id} in language: ${lang || 'default'}`);
                    return {
                        lang: langKey,
                        tests: [],
                        requested: plugin.numTests,
                        generated: 0,
                    };
                }
            });
            const languageResults = await Promise.allSettled(languagePromises);
            for (const result of languageResults) {
                if (result.status === 'fulfilled') {
                    const { lang, tests, requested, generated } = result.value;
                    allPluginTests.push(...tests);
                    resultsPerLanguage[lang || 'default'] = { requested, generated };
                }
                else {
                    // Handle rejected promise
                    logger_1.default.warn(`[Language Processing] Error generating tests for ${plugin.id}: ${result.reason}`);
                }
            }
            logger_1.default.debug(`[Language Processing] Total tests generated for ${plugin.id}: ${allPluginTests.length} (across ${languages.length} language(s))`);
            if (!Array.isArray(allPluginTests) || allPluginTests.length === 0) {
                logger_1.default.warn(`Failed to generate tests for ${plugin.id}`);
            }
            else {
                // Add metadata to each test case
                const testCasesWithMetadata = allPluginTests.map((t) => ({
                    ...t,
                    metadata: {
                        pluginId: plugin.id,
                        pluginConfig: resolvePluginConfig(plugin.config),
                        severity: plugin.severity ?? getPluginSeverity(plugin.id, resolvePluginConfig(plugin.config)),
                        modifiers: {
                            ...(testGenerationInstructions ? { testGenerationInstructions } : {}),
                            ...(plugin.config?.modifiers || {}),
                        },
                        ...(t?.metadata || {}),
                    },
                }));
                // Extract goal only when not already set (e.g. from single-call prompt+goal generation)
                const testsNeedingGoal = testCasesWithMetadata.filter((t) => t.metadata?.goal == null || t.metadata?.goal === '');
                if (testsNeedingGoal.length > 0) {
                    logger_1.default.debug(`Extracting goal for ${testsNeedingGoal.length} tests from ${plugin.id}...`);
                }
                for (const testCase of testCasesWithMetadata) {
                    if (testCase.metadata?.goal != null && testCase.metadata?.goal !== '') {
                        continue;
                    }
                    const key = injectVar ?? 'query';
                    const promptVar = testCase.vars?.[key];
                    const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar ?? '');
                    // For policy plugin, pass the policy text to improve intent extraction
                    const policy = getPolicyText(testCase.metadata);
                    const extractedGoal = await (0, util_2.extractGoalFromPrompt)(prompt, purpose, plugin.id, policy);
                    testCase.metadata.goal = extractedGoal;
                }
                // Add the results to main test cases array
                testCases.push(...testCasesWithMetadata);
            }
            if (showProgressBar) {
                progressBar?.increment(plugin.numTests * languages.length);
            }
            else {
                logger_1.default.info(`Generated ${allPluginTests.length} tests for ${plugin.id}`);
            }
            logger_1.default.debug(`Added ${allPluginTests.length} ${plugin.id} test cases`);
            // If multiple defined languages were used, create separate report entries for each language
            // Otherwise, use the aggregated result for the plugin
            const definedLanguages = languages.filter((lang) => lang !== undefined);
            if (definedLanguages.length > 1) {
                // Multiple languages - create separate entries for each
                // Don't show language suffix for English (en) - it's the default
                for (const [langKey, result] of Object.entries(resultsPerLanguage)) {
                    const displayId = langKey === 'en' ? plugin.id : `${plugin.id} (${langKey})`;
                    pluginResults[displayId] = result;
                }
            }
            else {
                // Single language or no language - use aggregated result
                pluginResults[plugin.id] = {
                    requested: plugin.id === 'intent' ? allPluginTests.length : plugin.numTests * languages.length,
                    generated: allPluginTests.length,
                };
            }
        }
        else if (plugin.id.startsWith('file://')) {
            try {
                const customPlugin = new custom_1.CustomPlugin(redteamProvider, purpose, injectVar ?? 'query', plugin.id);
                const customTests = await customPlugin.generateTests(plugin.numTests, delay);
                // Add metadata to each test case
                const testCasesWithMetadata = customTests.map((t) => ({
                    ...t,
                    metadata: {
                        pluginId: plugin.id,
                        pluginConfig: resolvePluginConfig(plugin.config),
                        severity: plugin.severity || getPluginSeverity(plugin.id, resolvePluginConfig(plugin.config)),
                        modifiers: {
                            ...(testGenerationInstructions ? { testGenerationInstructions } : {}),
                            ...(plugin.config?.modifiers || {}),
                        },
                        ...(t.metadata || {}),
                    },
                }));
                // Extract goal only when not already set (e.g. from single-call prompt+goal generation)
                const customTestsNeedingGoal = testCasesWithMetadata.filter((t) => t.metadata?.goal == null || t.metadata?.goal === '');
                if (customTestsNeedingGoal.length > 0) {
                    logger_1.default.debug(`Extracting goal for ${customTestsNeedingGoal.length} custom tests from ${plugin.id}...`);
                }
                for (const testCase of testCasesWithMetadata) {
                    if (testCase.metadata?.goal != null && testCase.metadata?.goal !== '') {
                        continue;
                    }
                    const key = injectVar ?? 'query';
                    const promptVar = testCase.vars?.[key];
                    const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar ?? '');
                    // For policy plugin, pass the policy text to improve intent extraction
                    const policy = getPolicyText(testCase.metadata);
                    const extractedGoal = await (0, util_2.extractGoalFromPrompt)(prompt, purpose, plugin.id, policy);
                    testCase.metadata.goal = extractedGoal;
                }
                // Add the results to main test cases array
                testCases.push(...testCasesWithMetadata);
                logger_1.default.debug(`Added ${customTests.length} custom test cases from ${plugin.id}`);
                pluginResults[plugin.id] = {
                    requested: plugin.numTests,
                    generated: customTests.length,
                };
            }
            catch (e) {
                logger_1.default.error(`Error generating tests for custom plugin ${plugin.id}: ${e}`);
                pluginResults[plugin.id] = { requested: plugin.numTests, generated: 0 };
            }
        }
        else {
            logger_1.default.warn(`Plugin ${plugin.id} not registered, skipping`);
            pluginResults[plugin.id] = { requested: plugin.numTests, generated: 0 };
            progressBar?.increment(plugin.numTests);
        }
    });
    // After generating plugin test cases but before applying strategies:
    const pluginTestCases = testCases;
    // Initialize strategy results
    const strategyResults = {};
    // Apply retry strategy first if it exists
    const retryStrategy = strategies.find((s) => s.id === 'retry');
    if (retryStrategy) {
        logger_1.default.debug('Applying retry strategy first');
        retryStrategy.config = {
            targetLabels,
            ...retryStrategy.config,
        };
        const { testCases: retryTestCases, strategyResults: retryResults } = await applyStrategies(pluginTestCases, [retryStrategy], injectVar);
        pluginTestCases.push(...retryTestCases);
        Object.assign(strategyResults, retryResults);
    }
    // Check for abort signal or apply non-basic strategies
    checkAbort();
    const { testCases: strategyTestCases, strategyResults: otherStrategyResults } = await applyStrategies(pluginTestCases, strategies.filter((s) => !['basic', 'retry'].includes(s.id)), injectVar, excludeTargetOutputFromAgenticAttackGeneration);
    Object.assign(strategyResults, otherStrategyResults);
    // Combine test cases based on basic strategy setting
    const finalTestCases = [...(includeBasicTests ? pluginTestCases : []), ...strategyTestCases];
    // Check for abort signal
    checkAbort();
    progressBar?.update({ task: 'Done.' });
    progressBar?.stop();
    if (progressBar) {
        // Newline after progress bar to avoid overlap
        logger_1.default.info('');
    }
    logger_1.default.info(generateReport(pluginResults, strategyResults));
    return { purpose, entities, testCases: finalTestCases, injectVar };
}
//# sourceMappingURL=index.js.map