"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __exportStar = (this && this.__exportStar) || function(m, exports) {
    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.EvalResultsFilterMode = exports.OutputFileExtension = exports.UnifiedConfigSchema = exports.TestSuiteConfigSchema = exports.TestSuiteSchema = exports.DerivedMetricSchema = exports.TestGeneratorConfigSchema = exports.AtomicTestCaseSchema = exports.ScenarioSchema = exports.TestCasesWithMetadataSchema = exports.TestCaseWithVarsFileSchema = exports.TestCaseSchema = exports.VarsSchema = exports.TestCasesWithMetadataPromptSchema = exports.AssertionSchema = exports.AssertionTypeSchema = exports.NotPrefixedAssertionTypesSchema = exports.SpecialAssertionTypesSchema = exports.VideoAssertionTypesSchema = exports.AudioAssertionTypesSchema = exports.BaseAssertionTypesSchema = exports.ResultFailureReason = exports.CompletedPromptSchema = exports.OutputConfigSchema = exports.CommandLineOptionsSchema = void 0;
exports.isResultFailureReason = isResultFailureReason;
exports.isGradingResult = isGradingResult;
// Note: This file is in the process of being deconstructed into `types/` and `validators/`
// Right now Zod and pure types are mixed together!
const zod_1 = require("zod");
const env_1 = require("../types/env");
const shared_1 = require("../types/shared");
const fileExtensions_1 = require("../util/fileExtensions");
const prompts_1 = require("../validators/prompts");
const providers_1 = require("../validators/providers");
const redteam_1 = require("../validators/redteam");
const shared_2 = require("../validators/shared");
__exportStar(require("../redteam/types"), exports);
__exportStar(require("./prompts"), exports);
__exportStar(require("./providers"), exports);
__exportStar(require("./shared"), exports);
__exportStar(require("./tracing"), exports);
exports.CommandLineOptionsSchema = zod_1.z.object({
    // Shared with TestSuite
    description: zod_1.z.string().optional(),
    prompts: zod_1.z.array(zod_1.z.string()).optional(),
    providers: zod_1.z.array(zod_1.z.string()),
    output: zod_1.z.any().optional(),
    // Shared with EvaluateOptions
    maxConcurrency: zod_1.z.coerce.number().int().positive().optional(),
    repeat: zod_1.z.coerce.number().int().positive().optional(),
    delay: zod_1.z.coerce.number().int().nonnegative().default(0),
    // Command line only
    vars: zod_1.z.string().optional(),
    tests: zod_1.z.string().optional(),
    config: zod_1.z.array(zod_1.z.string()).optional(),
    assertions: zod_1.z.string().optional(),
    modelOutputs: zod_1.z.string().optional(),
    transcripts: zod_1.z.string().optional(),
    verbose: zod_1.z.boolean().optional(),
    grader: zod_1.z.string().optional(),
    tableCellMaxLength: zod_1.z.coerce.number().int().positive().optional(),
    write: zod_1.z.boolean().optional(),
    cache: zod_1.z.boolean().optional(),
    table: zod_1.z.boolean().optional(),
    share: zod_1.z.boolean().optional(),
    noShare: zod_1.z.boolean().optional(),
    progressBar: zod_1.z.boolean().optional(),
    watch: zod_1.z.boolean().optional(),
    filterErrorsOnly: zod_1.z.string().optional(),
    filterFailing: zod_1.z.string().optional(),
    filterFirstN: zod_1.z.coerce.number().int().positive().optional(),
    filterMetadata: zod_1.z.string().optional(),
    filterPattern: zod_1.z.string().optional(),
    filterProviders: zod_1.z.string().optional(),
    filterSample: zod_1.z.coerce.number().int().positive().optional(),
    filterTargets: zod_1.z.string().optional(),
    var: zod_1.z.record(zod_1.z.string()).optional(),
    generateSuggestions: zod_1.z.boolean().optional(),
    promptPrefix: zod_1.z.string().optional(),
    promptSuffix: zod_1.z.string().optional(),
    retryErrors: zod_1.z.boolean().optional(),
    envPath: zod_1.z.string().optional(),
    application: zod_1.z
        .preprocess((val) => (Array.isArray(val) ? val : val ? [val] : []), zod_1.z.array(zod_1.z.string()))
        .optional(),
    applicationPath: zod_1.z
        .preprocess((val) => (Array.isArray(val) ? val : val ? [val] : []), zod_1.z.array(zod_1.z.string()))
        .optional(),
    library: zod_1.z.string().optional(),
    securityConfigId: zod_1.z.string().optional(),
    testRunId: zod_1.z.string().optional(),
    run: zod_1.z.boolean().optional(),
});
const GradingConfigSchema = zod_1.z.object({
    rubricPrompt: zod_1.z
        .union([
        zod_1.z.string(),
        zod_1.z.array(zod_1.z.string()),
        zod_1.z.array(zod_1.z.object({
            role: zod_1.z.string(),
            content: zod_1.z.string(),
        })),
    ])
        .optional(),
    provider: zod_1.z
        .union([zod_1.z.string(), zod_1.z.any(), zod_1.z.record(zod_1.z.string(), zod_1.z.union([zod_1.z.string(), zod_1.z.any()])).optional()])
        .optional(),
    factuality: zod_1.z
        .object({
        subset: zod_1.z.number().optional(),
        superset: zod_1.z.number().optional(),
        agree: zod_1.z.number().optional(),
        disagree: zod_1.z.number().optional(),
        differButFactual: zod_1.z.number().optional(),
    })
        .optional(),
});
exports.OutputConfigSchema = zod_1.z.object({
    /**
     * @deprecated in > 0.38.0. Use `transform` instead.
     */
    postprocess: zod_1.z.string().optional(),
    transform: zod_1.z.string().optional(),
    transformVars: zod_1.z.string().optional(),
    // The name of the variable to store the output of this test case
    storeOutputAs: zod_1.z.string().optional(),
});
const EvaluateOptionsSchema = zod_1.z.object({
    cache: zod_1.z.boolean().optional(),
    delay: zod_1.z.number().optional(),
    eventSource: zod_1.z.string().optional(),
    generateSuggestions: zod_1.z.boolean().optional(),
    /**
     * @deprecated This option has been removed as of 2024-08-21.
     * @description Use `maxConcurrency: 1` or the CLI option `-j 1` instead to run evaluations serially.
     * @author mldangelo
     */
    interactiveProviders: zod_1.z.boolean().optional(),
    maxConcurrency: zod_1.z.number().optional(),
    progressCallback: zod_1.z
        .function(zod_1.z.tuple([
        zod_1.z.number(),
        zod_1.z.number(),
        zod_1.z.number(),
        zod_1.z.custom(),
        zod_1.z.custom(),
    ]), zod_1.z.void())
        .optional(),
    repeat: zod_1.z.number().optional(),
    showProgressBar: zod_1.z.boolean().optional(),
    /**
     * Timeout in milliseconds for each individual test case/provider API call.
     * When reached, that specific test is marked as an error.
     * Default is 0 (no timeout).
     */
    timeoutMs: zod_1.z.number().optional(),
    /**
     * Maximum total runtime in milliseconds for the entire evaluation process.
     * When reached, all remaining tests are marked as errors and the evaluation ends.
     * Default is 0 (no limit).
     */
    maxEvalTimeMs: zod_1.z.number().optional(),
    testRunId: zod_1.z.string().optional(),
    isRedteam: zod_1.z.boolean().optional(),
    workspaceId: zod_1.z.string().optional(),
});
const PromptMetricsSchema = zod_1.z.object({
    score: zod_1.z.number(),
    testPassCount: zod_1.z.number(),
    testFailCount: zod_1.z.number(),
    testErrorCount: zod_1.z.number(),
    assertPassCount: zod_1.z.number(),
    assertFailCount: zod_1.z.number(),
    totalLatencyMs: zod_1.z.number(),
    tokenUsage: shared_1.BaseTokenUsageSchema,
    namedScores: zod_1.z.record(zod_1.z.string(), zod_1.z.number()),
    namedScoresCount: zod_1.z.record(zod_1.z.string(), zod_1.z.number()),
    redteam: zod_1.z
        .object({
        pluginPassCount: zod_1.z.record(zod_1.z.string(), zod_1.z.number()),
        pluginFailCount: zod_1.z.record(zod_1.z.string(), zod_1.z.number()),
        strategyPassCount: zod_1.z.record(zod_1.z.string(), zod_1.z.number()),
        strategyFailCount: zod_1.z.record(zod_1.z.string(), zod_1.z.number()),
    })
        .optional(),
    cost: zod_1.z.number(),
    testRunId: zod_1.z.string().optional(),
});
// Used for final prompt display
exports.CompletedPromptSchema = prompts_1.PromptSchema.extend({
    provider: zod_1.z.string(),
    metrics: PromptMetricsSchema.optional(),
});
exports.ResultFailureReason = {
    // The test passed, or we don't know exactly why the test case failed.
    NONE: 0,
    // The test case failed because an assertion rejected it.
    ASSERT: 1,
    // Test case failed due to some other error.
    ERROR: 2,
};
const validResultFailureReasons = new Set(Object.values(exports.ResultFailureReason));
function isResultFailureReason(value) {
    return validResultFailureReasons.has(value);
}
function isGradingResult(result) {
    return (typeof result === 'object' &&
        result !== null &&
        typeof result.pass === 'boolean' &&
        typeof result.score === 'number' &&
        typeof result.reason === 'string' &&
        (typeof result.namedScores === 'undefined' || typeof result.namedScores === 'object') &&
        (typeof result.tokensUsed === 'undefined' || typeof result.tokensUsed === 'object') &&
        (typeof result.componentResults === 'undefined' || Array.isArray(result.componentResults)) &&
        (typeof result.assertion === 'undefined' ||
            result.assertion === null ||
            typeof result.assertion === 'object') &&
        (typeof result.comment === 'undefined' || typeof result.comment === 'string'));
}
exports.BaseAssertionTypesSchema = zod_1.z.enum([
    'answer-relevance',
    'bleu',
    'classifier',
    'contains',
    'contains-all',
    'contains-any',
    'contains-html',
    'contains-json',
    'contains-sql',
    'contains-xml',
    'context-faithfulness',
    'context-recall',
    'context-relevance',
    'conversation-relevance',
    'cost',
    'equals',
    'factuality',
    'finish-reason',
    'g-eval',
    'gleu',
    'guardrails',
    'hallucination',
    'icontains',
    'icontains-all',
    'icontains-any',
    'is-html',
    'is-json',
    'not-is-json',
    'is-refusal',
    'is-sql',
    'is-valid-function-call',
    'is-valid-openai-function-call',
    'is-valid-openai-tools-call',
    'is-xml',
    'javascript',
    'latency',
    'levenshtein',
    'llm-rubric',
    'pi',
    'meteor',
    'model-graded-closedqa',
    'model-graded-factuality',
    'moderation',
    'perplexity',
    'perplexity-score',
    'python',
    'regex',
    'rouge-n',
    'not-rouge-n',
    'ruby',
    'similar',
    'similar:cosine',
    'similar:dot',
    'similar:euclidean',
    'starts-with',
    'tool-call-f1',
    'trace-error-spans',
    'trace-span-count',
    'trace-span-duration',
    'search-rubric',
    'webhook',
    'code-execution', // This is a custom assertion type for code execution
    'conversation-completeness',
    'knowledge-retention',
    'conversational-g-eval',
    'conversation-relevancy',
    'role-adherence',
    'code-correctness',
    'contains-code-language',
    'code-language',
    'code-readability',
    'hallucination',
    'image-helpfulness',
    'text-to-image',
    'image-editing',
    'multimodal-answer-relevancy',
    'image-coherence',
    'multimodal-faithfulness',
    'multimodal-g-eval',
    'image-reference',
    'multimodal-contextual-precision',
    'multimodal-contextual-recall',
    'multimodal-contextual-relevancy',
    'tool-correctness',
    'task-completion',
    'argument-correctness',
    'plan-quality',
    'plan-adherence',
    'goal-fulfillment',
    'execution-efficiency',
    'reasoning-coherence',
    'agent-custom-eval',
    'tool-selection-quality',
]);
exports.AudioAssertionTypesSchema = zod_1.z.enum([
    'audio-sentiment',
    'audio-call-category',
    'audio-interruption-handling',
    'audio-language',
    'audio-model-graded',
    'audio-intent-identification',
    'audio-consent-check',
    'audio-drop-out-rates',
    'audio-agent-response-time',
    'audio-talk-to-listen-ratio',
    'audio-conversation-coherence',
    'audio-disclaimer-check',
    'audio-pii-check',
    'audio-language-switch',
    'audio-professional-tone',
    'audio-clarity',
    'audio-background-noise',
    'audio-escalation',
    'audio-stt-accuracy',
    'audio-stt-correctness',
    'audio-call-disconnected',
    'audio-no-agent-response',
    'audio-incomplete-agent-response',
    'audio-repetition',
    'audio-greeting',
    'audio-number-correctness',
    'audio-stt-missing',
    'audio-call-outcome',
]);
exports.VideoAssertionTypesSchema = zod_1.z.enum([
    'video-visual-quality',
    'text-to-video',
    'video-factual-consistency',
    'video-temporal-consistency',
    'video-g-eval',
]);
exports.SpecialAssertionTypesSchema = zod_1.z.enum(['select-best', 'human', 'max-score']);
exports.NotPrefixedAssertionTypesSchema = exports.BaseAssertionTypesSchema.transform((baseType) => `not-${baseType}`);
exports.AssertionTypeSchema = zod_1.z.union([
    exports.BaseAssertionTypesSchema,
    exports.NotPrefixedAssertionTypesSchema,
    exports.SpecialAssertionTypesSchema,
    zod_1.z.custom(),
    exports.AudioAssertionTypesSchema,
    exports.VideoAssertionTypesSchema,
]);
// TODO(ian): maybe Assertion should support {type: config} to make the yaml cleaner
exports.AssertionSchema = zod_1.z.object({
    // Type of assertion
    type: exports.AssertionTypeSchema,
    // The expected value, if applicable
    value: zod_1.z.custom().optional(),
    // An external mapping of arbitrary strings to values that is passed
    // to the assertion for custom asserts
    config: zod_1.z.record(zod_1.z.string(), zod_1.z.any()).optional(),
    // The threshold value, only applicable for similarity (cosine distance)
    threshold: zod_1.z.number().optional(),
    // The weight of this assertion compared to other assertions in the test case. Defaults to 1.
    weight: zod_1.z.number().optional(),
    // Some assertions (similarity, llm-rubric) require an LLM provider
    provider: zod_1.z.custom().optional(),
    // Override the grading rubric
    rubricPrompt: zod_1.z.custom().optional(),
    // Tag this assertion result as a named metric
    metric: zod_1.z.string().optional(),
    // Process the output before running the assertion
    transform: zod_1.z.string().optional(),
    // The output value of this assertion. This is used to store the output of the assertion for later use.
    output: zod_1.z.string().optional(),
    // Extract context from the output using a transform
    contextTransform: zod_1.z.string().optional(),
});
const AssertionSetSchema = zod_1.z.object({
    type: zod_1.z.literal('assert-set'),
    // Sub assertions to be run for this assertion set
    assert: zod_1.z.array(zod_1.z.lazy(() => exports.AssertionSchema)),
    // The weight of this assertion compared to other assertions in the test case. Defaults to 1.
    weight: zod_1.z.number().optional(),
    // Tag this assertion result as a named metric
    metric: zod_1.z.string().optional(),
    // The required score for this assert set. If not provided, the test case is graded pass/fail.
    threshold: zod_1.z.number().optional(),
    // An external mapping of arbitrary strings to values that is defined
    // for every assertion in the set and passed into each assert
    config: zod_1.z.record(zod_1.z.string(), zod_1.z.any()).optional(),
});
// Used when building prompts index from files.
exports.TestCasesWithMetadataPromptSchema = zod_1.z.object({
    prompt: exports.CompletedPromptSchema,
    id: zod_1.z.string(),
    evalId: zod_1.z.string(),
});
const ProviderPromptMapSchema = zod_1.z.record(zod_1.z.string(), zod_1.z.union([zod_1.z.string().transform((value) => [value]), zod_1.z.array(zod_1.z.string())]));
// Metadata is a key-value store for arbitrary data
const MetadataSchema = zod_1.z.record(zod_1.z.string(), zod_1.z.any());
exports.VarsSchema = zod_1.z.record(zod_1.z.union([
    zod_1.z.string(),
    zod_1.z.number(),
    zod_1.z.boolean(),
    zod_1.z.array(zod_1.z.union([zod_1.z.string(), zod_1.z.number(), zod_1.z.boolean()])),
    zod_1.z.record(zod_1.z.string(), zod_1.z.any()),
    zod_1.z.array(zod_1.z.any()),
]));
// Each test case is graded pass/fail with a score.  A test case represents a unique input to the LLM after substituting `vars` in the prompt.
// HEADS UP: When you add a property here, you probably need to load it from `defaultTest` in evaluator.ts.
exports.TestCaseSchema = zod_1.z.object({
    // Optional description of what you're testing
    description: zod_1.z.string().optional(),
    // Key-value pairs to substitute in the prompt
    vars: exports.VarsSchema.optional(),
    // Override the provider.
    provider: zod_1.z.union([zod_1.z.string(), providers_1.ProviderOptionsSchema, providers_1.ApiProviderSchema]).optional(),
    // Output related from running values in Vars with provider. Having this value would skip running the prompt through the provider, and go straight to the assertions
    providerOutput: zod_1.z.union([zod_1.z.string(), zod_1.z.object({})]).optional(),
    providerResponse: zod_1.z.any().optional(),
    // Optional list of automatic checks to run on the LLM output
    assert: zod_1.z.array(zod_1.z.union([AssertionSetSchema, exports.AssertionSchema])).optional(),
    // Optional scoring function to run on the LLM output
    assertScoringFunction: zod_1.z
        .union([
        zod_1.z
            .string()
            .regex(new RegExp(`^file://.*\\.(${fileExtensions_1.JAVASCRIPT_EXTENSIONS?.join('|')}|py)(?::[\\w.]+)?$`)),
        zod_1.z.custom(),
    ])
        .optional(),
    // Additional configuration settings for the prompt
    options: zod_1.z
        .intersection(zod_1.z.intersection(prompts_1.PromptConfigSchema, exports.OutputConfigSchema), zod_1.z.intersection(GradingConfigSchema, zod_1.z.object({
        // If true, do not expand arrays of variables into multiple eval cases.
        disableVarExpansion: zod_1.z.boolean().optional(),
        // If true, do not include an implicit `_conversation` variable in the prompt.
        disableConversationVar: zod_1.z.boolean().optional(),
        // If true, run this without concurrency no matter what
        runSerially: zod_1.z.boolean().optional(),
    })))
        .optional(),
    // The required score for this test case.  If not provided, the test case is graded pass/fail.
    threshold: zod_1.z.number().optional(),
    metadata: zod_1.z
        .intersection(MetadataSchema, zod_1.z.object({
        pluginConfig: zod_1.z.custom().optional(),
        strategyConfig: zod_1.z.custom().optional(),
    }))
        .optional(),
});
exports.TestCaseWithVarsFileSchema = exports.TestCaseSchema.extend({
    vars: zod_1.z.union([exports.VarsSchema, zod_1.z.string(), zod_1.z.array(zod_1.z.string())]).optional(),
});
exports.TestCasesWithMetadataSchema = zod_1.z.object({
    id: zod_1.z.string(),
    testCases: zod_1.z.union([zod_1.z.string(), zod_1.z.array(zod_1.z.union([zod_1.z.string(), exports.TestCaseSchema]))]),
    recentEvalDate: zod_1.z.date(),
    recentEvalId: zod_1.z.string(),
    count: zod_1.z.number(),
    prompts: zod_1.z.array(exports.TestCasesWithMetadataPromptSchema),
});
exports.ScenarioSchema = zod_1.z.object({
    // Optional description of what you're testing
    description: zod_1.z.string().optional(),
    // Default test case config
    config: zod_1.z.array(exports.TestCaseSchema.partial()),
    // Optional list of automatic checks to run on the LLM output
    tests: zod_1.z.array(exports.TestCaseSchema),
});
// Same as a TestCase, except the `vars` object has been flattened into its final form.
exports.AtomicTestCaseSchema = exports.TestCaseSchema.extend({
    vars: zod_1.z.record(zod_1.z.union([zod_1.z.string(), zod_1.z.object({})])).optional(),
}).strict();
/**
 * Configuration schema for test generators that accept parameters
 *
 * @example
 * ```yaml
 * tests:
 *   - path: file://test_cases.py:generate_tests
 *     config:
 *       dataset: truthfulqa
 *       split: validation
 *       max_rows: 100
 * ```
 */
exports.TestGeneratorConfigSchema = zod_1.z.object({
    /** Path to the test generator function (e.g., file://path/to/tests.py:function_name) */
    path: zod_1.z.string(),
    /**
     * Configuration object passed to the generator function
     * Common configuration options include:
     * - dataset: string - Dataset identifier
     * - split: string - Dataset split (train/validation/test)
     * - max_rows: number - Maximum number of test cases to generate
     * - languages: string[] - Array of target languages
     * - difficulty: string - Difficulty level (basic/intermediate/advanced)
     * - categories: string[] - Array of test categories
     * - data: object - Custom data configuration
     *
     * Values can reference external files using file:// paths
     */
    config: zod_1.z
        .record(zod_1.z.string(), zod_1.z.union([
        zod_1.z.string(),
        zod_1.z.number(),
        zod_1.z.boolean(),
        zod_1.z.array(zod_1.z.union([zod_1.z.string(), zod_1.z.number(), zod_1.z.boolean()])),
        zod_1.z.record(zod_1.z.string(), zod_1.z.any()),
        zod_1.z.any(), // Allow for complex nested structures and file:// references
    ]))
        .optional(),
});
exports.DerivedMetricSchema = zod_1.z.object({
    // The name of this metric
    name: zod_1.z.string(),
    // The function to calculate the metric - either a mathematical expression or a function that takes the scores and returns a number
    value: zod_1.z.union([
        zod_1.z.string(),
        zod_1.z
            .function()
            .args(zod_1.z.record(zod_1.z.string(), zod_1.z.number()), zod_1.z.custom())
            .returns(zod_1.z.number()),
    ]),
});
// The test suite defines the "knobs" that we are tuning in prompt engineering: providers and prompts
exports.TestSuiteSchema = zod_1.z.object({
    // Optional tags to describe the test suite
    tags: zod_1.z.record(zod_1.z.string(), zod_1.z.string()).optional(),
    // Optional description of what your LLM is trying to do
    description: zod_1.z.string().optional(),
    // One or more LLM APIs to use
    providers: zod_1.z.array(providers_1.ApiProviderSchema),
    // One or more prompt strings
    prompts: zod_1.z.array(prompts_1.PromptSchema),
    // Optional mapping of provider to prompt display strings.  If not provided,
    // all prompts are used for all providers.
    providerPromptMap: ProviderPromptMapSchema.optional(),
    // Test cases
    tests: zod_1.z.array(exports.TestCaseSchema).optional(),
    // scenarios
    scenarios: zod_1.z.array(exports.ScenarioSchema).optional(),
    // Sets the default properties for each test case. Useful for setting an assertion, on all test cases, for example.
    defaultTest: zod_1.z
        .union([
        zod_1.z.string().refine((val) => val.startsWith('file://'), {
            message: 'defaultTest string must start with file://',
        }),
        zod_1.z.any(),
        exports.TestCaseSchema.omit({ description: true }),
    ])
        .optional(),
    // Nunjucks filters
    nunjucksFilters: shared_2.NunjucksFilterMapSchema.optional(),
    // Envar overrides
    env: env_1.ProviderEnvOverridesSchema.optional(),
    // Metrics to calculate after the eval has been completed
    derivedMetrics: zod_1.z.array(exports.DerivedMetricSchema).optional(),
    // Extensions that are called at various plugin points
    extensions: zod_1.z
        .array(zod_1.z
        .string()
        .refine((value) => value.startsWith('file://'), {
        message: 'Extension must start with file://',
    })
        .refine((value) => {
        const parts = value.split(':');
        return parts.length === 3 && parts.every((part) => part.trim() !== '');
    }, {
        message: 'Extension must be of the form file://path/to/file.py:function_name',
    })
        .refine((value) => {
        const parts = value.split(':');
        return ((parts[1].endsWith('.py') || (0, fileExtensions_1.isJavascriptFile)(parts[1])) &&
            (parts.length === 3 || parts.length === 2));
    }, {
        message: 'Extension must be a python (.py) or javascript (.js, .ts, .mjs, .cjs, etc.) file followed by a colon and function name',
    }))
        .nullable()
        .optional(),
    // Redteam configuration - used only when generating redteam tests
    redteam: zod_1.z.custom().optional(),
    parentPromptId: zod_1.z.string().optional(),
    aiSpeaksFirst: zod_1.z.boolean().optional(),
    localProvider: zod_1.z.boolean().optional(),
    // Tracing configuration (simplified version for parsed TestSuite)
    tracing: zod_1.z
        .object({
        enabled: zod_1.z.boolean(),
        otlp: zod_1.z
            .object({
            http: zod_1.z
                .object({
                enabled: zod_1.z.boolean(),
                port: zod_1.z.number(),
                host: zod_1.z.string().optional(),
                acceptFormats: zod_1.z.array(zod_1.z.string()),
            })
                .optional(),
            grpc: zod_1.z
                .object({
                enabled: zod_1.z.boolean(),
                port: zod_1.z.number(),
            })
                .optional(),
        })
            .optional(),
        storage: zod_1.z
            .object({
            type: zod_1.z.string(),
            retentionDays: zod_1.z.number(),
        })
            .optional(),
        forwarding: zod_1.z
            .object({
            enabled: zod_1.z.boolean(),
            endpoint: zod_1.z.string(),
            headers: zod_1.z.record(zod_1.z.string()).optional(),
        })
            .optional(),
    })
        .optional(),
    isTextConversationPrompt: zod_1.z.boolean().optional(),
});
// TestSuiteConfig = Test Suite, but before everything is parsed and resolved.  Providers are just strings, prompts are filepaths, tests can be filepath or inline.
exports.TestSuiteConfigSchema = zod_1.z.object({
    // Optional tags to describe the test suite
    tags: zod_1.z.record(zod_1.z.string(), zod_1.z.string()).optional(),
    // Optional description of what you're trying to test
    description: zod_1.z.string().optional(),
    projectId: zod_1.z.string().optional(),
    applicationId: zod_1.z.string().optional(),
    sessionId: zod_1.z.string().optional(),
    workspaceId: zod_1.z.string().optional(),
    orgId: zod_1.z.string().optional(),
    // One or more LLM APIs to use, for example: openai:gpt-4o-mini, openai:gpt-4o, localai:chat:vicuna
    providers: providers_1.ProvidersSchema,
    // One or more prompt files to load
    prompts: zod_1.z.union([
        zod_1.z.string(),
        zod_1.z.array(zod_1.z.union([
            zod_1.z.string(),
            zod_1.z.object({
                id: zod_1.z.string(),
                label: zod_1.z.string().optional(),
                raw: zod_1.z.string().optional(),
            }),
            prompts_1.PromptSchema,
        ])),
        zod_1.z.record(zod_1.z.string(), zod_1.z.string()),
    ]),
    // Path to a test file, OR list of LLM prompt variations (aka "test case")
    tests: zod_1.z
        .union([
        zod_1.z.string(),
        zod_1.z.array(zod_1.z.union([zod_1.z.string(), exports.TestCaseSchema, exports.TestGeneratorConfigSchema])),
        exports.TestGeneratorConfigSchema,
    ])
        .optional(),
    // Scenarios, groupings of data and tests to be evaluated
    scenarios: zod_1.z.array(zod_1.z.union([zod_1.z.string(), exports.ScenarioSchema])).optional(),
    // Sets the default properties for each test case. Useful for setting an assertion, on all test cases, for example.
    defaultTest: zod_1.z
        .union([
        zod_1.z.string().refine((val) => val.startsWith('file://'), {
            message: 'defaultTest string must start with file://',
        }),
        exports.TestCaseSchema.omit({ description: true }),
    ])
        .optional(),
    // Path to write output. Writes to console/web viewer if not set.
    outputPath: zod_1.z.union([zod_1.z.string(), zod_1.z.array(zod_1.z.string())]).optional(),
    // Determines whether or not sharing is enabled.
    sharing: zod_1.z
        .union([
        zod_1.z.boolean(),
        zod_1.z.object({
            apiBaseUrl: zod_1.z.string().optional(),
            appBaseUrl: zod_1.z.string().optional(),
        }),
    ])
        .optional(),
    // Nunjucks filters
    nunjucksFilters: zod_1.z.record(zod_1.z.string(), zod_1.z.string()).optional(),
    // Envvar overrides
    env: zod_1.z
        .union([
        env_1.ProviderEnvOverridesSchema,
        zod_1.z.record(zod_1.z.string(), zod_1.z.union([
            zod_1.z.string(),
            zod_1.z.number().transform((n) => String(n)),
            zod_1.z.boolean().transform((b) => String(b)),
        ])),
    ])
        .optional(),
    // Metrics to calculate after the eval has been completed
    derivedMetrics: zod_1.z.array(exports.DerivedMetricSchema).optional(),
    // Extension that is called at various plugin points
    extensions: zod_1.z.array(zod_1.z.string()).nullable().optional(),
    // Any other information about this configuration.
    metadata: MetadataSchema.optional(),
    // Redteam configuration - used only when generating redteam tests
    redteam: redteam_1.RedteamConfigSchema.optional(),
    // Write results to disk so they can be viewed in web viewer
    writeLatestResults: zod_1.z.boolean().optional(),
    turns: zod_1.z.any().optional(),
    conversationFlow: zod_1.z.any().optional(),
    parentPromptId: zod_1.z.string().optional(),
    localProvider: zod_1.z.boolean().optional(),
    // Tracing configuration
    tracing: zod_1.z
        .object({
        enabled: zod_1.z.boolean().default(false),
        // OTLP receiver configuration
        otlp: zod_1.z
            .object({
            http: zod_1.z
                .object({
                enabled: zod_1.z.boolean().default(true),
                port: zod_1.z.number().default(4318),
                host: zod_1.z.string().default('0.0.0.0'),
                acceptFormats: zod_1.z.array(zod_1.z.enum(['protobuf', 'json'])).default(['json']),
            })
                .optional(),
            grpc: zod_1.z
                .object({
                enabled: zod_1.z.boolean().default(false),
                port: zod_1.z.number().default(4317),
            })
                .optional(),
        })
            .optional(),
        // Storage configuration
        storage: zod_1.z
            .object({
            type: zod_1.z.enum(['sqlite']).default('sqlite'),
            retentionDays: zod_1.z.number().default(30),
        })
            .optional(),
        // Optional: Forward traces to another collector
        forwarding: zod_1.z
            .object({
            enabled: zod_1.z.boolean().default(false),
            endpoint: zod_1.z.string(),
            headers: zod_1.z.record(zod_1.z.string()).optional(),
        })
            .optional(),
    })
        .optional(),
});
exports.UnifiedConfigSchema = exports.TestSuiteConfigSchema.extend({
    evaluateOptions: EvaluateOptionsSchema.optional(),
    commandLineOptions: exports.CommandLineOptionsSchema.partial().optional(),
    providers: providers_1.ProvidersSchema.optional(),
    targets: providers_1.ProvidersSchema.optional(),
})
    .refine((data) => {
    const hasTargets = Boolean(data.targets);
    const hasProviders = Boolean(data.providers);
    return (hasTargets && !hasProviders) || (!hasTargets && hasProviders);
}, {
    message: "Exactly one of 'targets' or 'providers' must be provided, but not both",
})
    .transform((data) => {
    if (data.targets && !data.providers) {
        data.providers = data.targets;
        delete data.targets;
    }
    // Handle null extensions, undefined extensions, or empty arrays by deleting the field
    if (data.extensions === null ||
        data.extensions === undefined ||
        (Array.isArray(data.extensions) && data.extensions.length === 0)) {
        delete data.extensions;
    }
    return data;
});
// Custom provider schema (either API URL or Webhook)
const CustomProviderSchema = zod_1.z
    .object({
    type: zod_1.z.literal('custom'),
    apiUrl: zod_1.z.string().url().optional(),
    webhook: zod_1.z.string().url().optional(),
})
    .refine((data) => data.apiUrl || data.webhook, {
    message: "Either 'apiUrl' or 'webhook' must be provided for custom provider",
});
// LLM provider schema (this schema should match the one defined in ProvidersSchema)
const LLMProviderSchema = providers_1.ProvidersSchema; // Assuming ProvidersSchema is already defined elsewhere
// Combined provider configuration schema
const _ProvidersConfigSchema = zod_1.z
    .object({
    provider_type: zod_1.z.enum(['llm', 'custom', 'web_socket']),
    config: zod_1.z.union([LLMProviderSchema, CustomProviderSchema]),
})
    .refine((data) => {
    // Ensure that if the provider type is "custom", the config is of the "custom" type, and vice versa
    // if (data.provider_type === 'llm' && data.provider_config.type === 'custom') {
    //   return false; // Invalid if type doesn't match the configuration
    // }
    // if (data.provider_type === 'custom' && data.provider_config.type !== 'custom') {
    //   return false; // Invalid if type doesn't match the configuration
    // }
    return true;
}, {
    message: 'Provider config type does not match provider type',
});
const _PrompTests = zod_1.z.array(zod_1.z.object({
    description: zod_1.z.string(),
    vars: zod_1.z.record(zod_1.z.string(), zod_1.z.string()),
    assert: zod_1.z.array(zod_1.z.object({
        type: zod_1.z.string(),
        value: zod_1.z.string(),
    })),
}));
// used for writing eval results
exports.OutputFileExtension = zod_1.z.enum([
    'csv',
    'html',
    'json',
    'jsonl',
    'txt',
    'xml',
    'yaml',
    'yml',
]);
exports.EvalResultsFilterMode = zod_1.z.enum([
    'all',
    'failures',
    'different',
    'highlights',
    'errors',
    'passes',
]);
//# sourceMappingURL=index.js.map