"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.assertionsSeed = void 0;
exports.seedAssertions = seedAssertions;
const uuid_1 = require("uuid");
const index_1 = require("../index");
const tables_1 = require("../tables");
const inferProviderResponseRequirements_1 = require("./inferProviderResponseRequirements");
exports.assertionsSeed = [
    // Text Matching Assertions
    {
        name: 'equals',
        label: 'Expected Output',
        description: 'Checks if the output exactly matches the expected value',
        group: 'text-matching',
        valueType: 'string',
        placeholder: 'e.g. 15',
        helpText: 'Enter the exact value you expect the output to match.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'contains',
        label: 'Expected Output Contains',
        description: 'Checks if the output contains the expected value',
        group: 'text-matching',
        valueType: 'string',
        placeholder: 'e.g. hello world',
        helpText: 'Enter a text fragment you expect to be present in the output.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'icontains',
        label: 'Expected Output Contains',
        description: 'Case-insensitive check if the output contains the expected value',
        group: 'text-matching',
        valueType: 'string',
        placeholder: "e.g. 'Hello' will match 'hello'",
        helpText: 'Enter a text fragment to check for in the output (case-insensitive).',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'contains-all',
        label: 'Expected Output Contains',
        description: 'Checks if the output contains all of the expected values',
        group: 'text-matching',
        valueType: 'array',
        placeholder: 'e.g. apple',
        helpText: 'Enter all text fragments that must be present in the output.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'contains-any',
        label: 'Expected Output Contains',
        description: 'Checks if the output contains any of the expected values',
        group: 'text-matching',
        valueType: 'array',
        placeholder: 'e.g. success',
        helpText: 'Enter any potential text fragments you expect in the output.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'starts-with',
        label: 'Expected Output Starts with',
        description: 'Checks if the output starts with the expected value',
        group: 'text-matching',
        valueType: 'string',
        placeholder: 'e.g. The answer is',
        helpText: 'Enter the text you expect at the beginning of the output.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'regex',
        label: 'Expected Output Matches Regex',
        description: 'Checks if the output matches the regular expression pattern',
        group: 'text-matching',
        valueType: 'regex',
        placeholder: 'e.g. ^[0-9]+$',
        helpText: 'Enter a regex pattern to validate the output against.',
        mandatory: true,
        additionalFields: [],
    },
    // Format Validation Assertions
    {
        name: 'is-json',
        label: 'Expected Output',
        description: 'Validates if the output is valid JSON',
        group: 'format-validation',
        valueType: 'array',
        placeholder: 'e.g. user_name',
        helpText: 'Enter expected JSON keys or leave empty to validate any JSON.',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'contains-json',
        label: '',
        description: 'Checks if the output contains valid JSON',
        group: 'format-validation',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'is-xml',
        label: 'Expected Output',
        description: 'Validates if the output is valid XML',
        group: 'format-validation',
        valueType: 'array',
        placeholder: 'e.g., book, author',
        helpText: 'Enter expected XML elements or leave empty to validate any XML.',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'contains-xml',
        label: '',
        description: 'Checks if the output contains valid XML',
        group: 'format-validation',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'is-sql',
        label: '',
        description: 'Validates if the output is valid SQL',
        group: 'format-validation',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [
            {
                name: 'databaseType',
                type: 'single-select',
                label: 'Database Type',
                options: [
                    'Athena',
                    'BigQuery',
                    'DB2',
                    'FlinkSQL',
                    'Hive',
                    'MariaDB',
                    'MySQL',
                    'Noql',
                    'PostgresQL',
                    'Redshift',
                    'Snowflake(alpha)',
                    'Sqlite',
                    'TransactSQL',
                ],
                placeholder: 'SelectSelect',
                helpText: 'Select the database type for SQL validation.',
                description: 'Select the database type for SQL validation.',
                mandatory: false,
            },
            {
                name: 'allowedTables',
                type: 'array',
                label: 'Allowed Tables',
                placeholder: "e.g. ['users', 'orders']",
                helpText: 'Enter table names that are allowed in the SQL query.',
                description: 'Enter table names that are allowed in the SQL query.',
                mandatory: false,
            },
            {
                name: 'allowedColumns',
                type: 'array',
                label: 'Allowed Columns',
                placeholder: "e.g. ['id', 'name', 'email']",
                helpText: 'Enter column names that are allowed in the SQL query.',
                description: 'Enter column names that are allowed in the SQL query.',
                mandatory: false,
            },
        ],
    },
    {
        name: 'contains-sql',
        description: 'Checks if the output contains valid SQL',
        group: 'format-validation',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    // Semantic Similarity Assertions
    {
        name: 'similar',
        label: 'Reference Text',
        description: 'Checks semantic similarity between output and expected value',
        group: 'factuality-evaluation',
        valueType: 'string',
        placeholder: "e.g. 'Paris is the capital of France'",
        helpText: 'Enter a reference text to check for semantic similarity against the output.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'llm-rubric',
        label: 'Grading Criteria',
        description: 'Evaluates output against a rubric using LLM',
        group: 'llm-judgment-metrics',
        valueType: 'string',
        placeholder: "e.g. 'Accuracy: 5 points, Clarity: 5 points'",
        helpText: 'Specify the rubric criteria for LLM evaluation.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'model-graded-closedqa',
        label: 'Grading Criteria',
        description: 'Evaluates closed QA responses using model grading',
        group: 'llm-judgment-metrics',
        valueType: 'string',
        placeholder: "e.g. 'Correctness, Relevance, Completeness'",
        helpText: 'Define the criteria for grading closed QA responses.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'factuality',
        label: 'Reference Text',
        description: 'Checks factual accuracy of the output',
        group: 'factuality-evaluation',
        valueType: 'string',
        placeholder: "e.g., 'The Earth orbits around the Sun'",
        helpText: 'Enter the factual statement to verify against the output.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'g-eval',
        label: 'Evaluation Criteria',
        description: 'Evaluates using G-Eval metric',
        group: 'llm-judgment-metrics',
        valueType: 'array',
        //options: ['relevance', 'coherence', 'fluency', 'consistency'],
        placeholder: 'e.g., "Evaluate the response for fluency, coherence, and relevance."',
        helpText: 'Specify the criteria for the G-Eval evaluation',
        mandatory: true,
        additionalFields: [],
    },
    {
    /*
    name: 'select-best',
    label: 'Grading Criteria',
    description: 'Selects best response from multiple options',
    group: 'llm-judgment-metrics',
    valueType: 'string',
    placeholder: 'e.g., "Choose the most concise and accurate response"',
    helpText: 'Enter criteria to evaluates each output against.',
    mandatory: true,
    additionalFields: [],
  */
    },
    {
        name: 'hallucination',
        label: '',
        description: 'Detects hallucinations or unsupported content in generated output based on source context',
        group: 'factuality-evaluation',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: true,
        additionalFields: [
            {
                name: 'context',
                type: 'string',
                label: 'Context',
                helpText: 'Enter the context retrieved from knowledge base',
                mandatory: true,
                description: '',
                placeholder: '',
            },
            {
                name: 'threshold',
                type: 'number',
                label: 'Minimum Hallucination Score',
                helpText: 'Enter the minimum hallucination score',
                mandatory: false,
                description: 'Minimum allowed hallucination score',
                placeholder: '0.7',
                defaultValue: 0.7,
            },
        ],
    },
    // Metrics Assertions
    {
        name: 'bleu',
        label: 'Reference Text',
        description: 'Calculates BLEU score for text similarity',
        group: 'metrics',
        valueType: 'string',
        placeholder: "e.g. 'Climate change is a global challenge'",
        helpText: 'Enter reference text to calculate the BLEU score',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'rouge-n',
        label: 'Reference Text',
        description: 'Calculates ROUGE-N score for text similarity',
        group: 'metrics',
        valueType: 'string',
        placeholder: "e.g. 'Climate change is a global challenge'",
        helpText: 'Enter reference text to calculate the ROUGE-N score',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'latency',
        label: '',
        description: 'Checks response latency against threshold',
        group: 'metrics',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [
            {
                name: 'threshold',
                type: 'number',
                placeholder: '1000',
                helpText: 'Enter the maximum latency in milliseconds',
                label: 'Maximum Latency (ms)',
                description: 'Maximum allowed latency in milliseconds',
                defaultValue: 1000,
                mandatory: true,
            },
        ],
    },
    {
        name: 'perplexity',
        label: '',
        description: 'Calculates perplexity score',
        group: 'metrics',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [
            {
                name: 'threshold',
                type: 'number',
                placeholder: '1.5',
                helpText: 'Enter the maximum perplexity score',
                label: 'Maximum Perplexity',
                description: 'Maximum allowed perplexity score',
                defaultValue: 1.5,
                mandatory: false,
            },
        ],
    },
    {
        name: 'perplexity-score',
        label: '',
        description: 'Calculates normalized perplexity score',
        group: 'metrics',
        valueType: 'number',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [
            {
                name: 'threshold',
                type: 'number',
                placeholder: '0.5',
                helpText: 'Enter the maximum normalized perplexity score',
                label: 'Maximum Perplexity Score',
                description: 'Maximum allowed normalized perplexity score',
                defaultValue: 0.5,
                mandatory: false,
            },
        ],
    },
    {
        name: 'cost',
        label: 'Threshold',
        description: 'Checks API call cost against threshold',
        group: 'metrics',
        valueType: 'number',
        placeholder: '0.01',
        helpText: 'Enter the maximum cost in dollars',
        mandatory: true,
        additionalFields: [],
    },
    // Rag Metrics
    {
        name: 'answer-relevance',
        label: '',
        description: 'Evaluates answer relevance to question',
        group: 'rag-metrics',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'context-faithfulness',
        label: 'Query',
        description: 'Checks if output only makes claims that are supported by the provided context',
        group: 'rag-metrics',
        valueType: 'string',
        placeholder: '',
        helpText: 'Enter the query that is sent to vector DB to get the context',
        mandatory: false,
        additionalFields: [
            {
                name: 'context',
                type: 'string',
                label: 'Context',
                helpText: 'Enter the context retrieved from knowledge base',
                mandatory: false,
                description: '',
                placeholder: '',
            },
            {
                name: 'threshold',
                type: 'number',
                label: 'Minimum Context Faithfulness Score',
                helpText: 'Enter the minimum context faithfulness score',
                mandatory: false,
                description: 'Minimum allowed context faithfulness score',
                placeholder: '0.7',
                defaultValue: 0.7,
            },
        ],
    },
    {
        name: 'context-recall',
        label: 'Query',
        description: 'Evaluates if the retrieved context contains right information needed to generate a known correct answer',
        group: 'rag-metrics',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [
            {
                name: 'context',
                type: 'string',
                label: 'Context',
                helpText: 'Enter the context retrieved from knowledge base',
                mandatory: false,
                description: '',
                placeholder: '',
            },
            {
                name: 'groundTruth',
                type: 'string',
                label: 'Ground Truth',
                helpText: 'Enter the expected answer/ground truth ',
                mandatory: false,
                description: '',
                placeholder: '',
            },
            {
                name: 'threshold',
                type: 'number',
                label: 'Minimum Context Recall Score',
                helpText: 'Enter the minimum context recall score',
                mandatory: false,
                description: 'Minimum allowed context recall score',
                placeholder: '0.7',
                defaultValue: 0.7,
            },
        ],
    },
    {
        name: 'context-relevance',
        label: 'Query',
        description: 'Evaluates the fraction of retrieved context that is minimally needed to answer the prompt.',
        group: 'rag-metrics',
        valueType: 'string',
        placeholder: '',
        helpText: 'Enter the query that is sent to vector DB to get the context',
        mandatory: false,
        additionalFields: [
            {
                name: 'context',
                type: 'string',
                placeholder: '',
                helpText: 'Enter the context retrieved from knowledge base',
                label: 'Context',
                description: 'Enter the context retrieved from knowledge base',
                mandatory: false,
            },
            {
                name: 'threshold',
                type: 'number',
                label: 'Minimum Context Relevance Score',
                helpText: 'Enter the minimum context relevance score',
                mandatory: false,
                description: 'Minimum allowed context relevance score',
                placeholder: '0.3',
                defaultValue: 0.3,
            },
        ],
    },
    // Content Moderation
    {
        name: 'moderation',
        label: 'Category',
        description: 'Checks content for inappropriate material',
        group: 'content-moderation',
        valueType: 'multiselect',
        options: ['hate', 'harassment', 'self-harm', 'sexual', 'violence'],
        placeholder: 'Select',
        helpText: 'Select categories to be monitored',
        mandatory: false,
        additionalFields: [],
    },
    // Audio Analysis
    {
        name: 'audio-sentiment',
        label: 'Expected Sentiment',
        description: 'Analyzes sentiment in audio content',
        group: 'audio-analysis',
        valueType: 'single-select',
        mandatory: false,
        options: ['positive', 'negative', 'neutral'],
        placeholder: 'Select',
        helpText: 'Select the expected sentiment of your output.',
        additionalFields: [],
    },
    {
        name: 'audio-call-category',
        label: '',
        description: 'Categorizes audio call type',
        group: 'audio-analysis',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: true,
        additionalFields: [],
        // allowEval: 0,
    },
    {
        name: 'audio-language',
        label: 'Expected Language',
        description: 'Detects language in audio',
        group: 'audio-analysis',
        options: ['english', 'spanish', 'french', 'german', 'chinese', 'japanese'],
        valueType: 'single-select',
        mandatory: false,
        placeholder: 'Select',
        helpText: 'Select the expected languages in the audio.',
        additionalFields: [],
        // allowEval: 0,
    },
    {
        name: 'audio-interruption-handling',
        label: '',
        description: 'Evaluates handling of interruptions in audio',
        group: 'audio-analysis',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'audio-intent-identification',
        label: '',
        description: 'Checks for false intent identification in audio',
        group: 'audio-analysis',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'audio-consent-check',
        label: 'Conditions before the consent is taken',
        description: 'Verifies consent in audio interactions',
        group: 'audio-analysis',
        valueType: 'string',
        placeholder: 'e.g., Before asking for personal information',
        helpText: 'Enter the specific conditions that must be met before consent is obtained.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'audio-drop-out-rates',
        label: '',
        description: 'Measures audio drop-out rates',
        group: 'audio-analysis',
        valueType: 'number',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'audio-agent-response-time',
        label: '',
        description: 'Measures agent response time in audio',
        group: 'audio-analysis',
        valueType: 'number',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'audio-talk-to-listen-ratio',
        label: '',
        description: 'Calculates talk-to-listen ratio in audio',
        group: 'audio-analysis',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'audio-conversation-coherence',
        label: '',
        description: 'Evaluates conversation coherence in audio',
        group: 'audio-analysis',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'audio-disclaimer-check',
        label: 'Conditions before the consent is taken',
        description: 'Verifies presence of disclaimers in audio',
        group: 'audio-analysis',
        valueType: 'string',
        placeholder: 'e.g., Before asking for personal information',
        helpText: 'Enter the specific conditions when disclaimers must be provided.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'audio-language-switch',
        label: '',
        description: 'Detects language switches in audio',
        group: 'audio-analysis',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'audio-background-noise',
        label: '',
        description: 'Detects background noise in audio',
        group: 'audio-analysis',
        valueType: 'single-select',
        mandatory: false,
        options: ['low', 'high', 'medium'],
        placeholder: 'Select',
        helpText: 'Select the expected background noise in the audio.',
        additionalFields: [],
    },
    {
        name: 'audio-call-outcome',
        label: 'Expected Call Outcome',
        description: 'Detects call outcome in audio',
        group: 'audio-analysis',
        valueType: 'textarea',
        mandatory: false,
        placeholder: '',
        helpText: 'Select the expected call outcome in the audio.',
        additionalFields: [],
        // allowEval: 0,
    },
    {
        name: 'audio-greeting',
        label: 'Expected Greeting',
        description: 'Detects greeting in audio',
        group: 'audio-analysis',
        valueType: 'string',
        mandatory: false,
        placeholder: '',
        helpText: 'Select the expected greeting in the audio.',
        additionalFields: [],
        // allowObserve: 0,
    },
    {
        name: 'audio-repetition',
        label: '',
        description: 'Detects repetition in audio',
        group: 'audio-analysis',
        valueType: 'none',
        mandatory: false,
        placeholder: '',
        helpText: '',
        additionalFields: [],
        // allowObserve: 0,
    },
    {
        name: 'audio-clarity',
        label: '',
        description: 'Evaluates audio call clarity',
        group: 'audio-analysis',
        valueType: 'none',
        mandatory: false,
        placeholder: '',
        helpText: '',
        additionalFields: [],
        // allowObserve: 0,
    },
    {
        name: 'audio-no-agent-response',
        label: '',
        description: 'Detects if the AI agent did not respond in audio',
        group: 'audio-analysis',
        valueType: 'none',
        mandatory: false,
        placeholder: '',
        helpText: '',
        additionalFields: [],
        // allowObserve: 0,
    },
    {
        name: 'audio-incomplete-agent-response',
        label: '',
        description: 'Detects if the AI agent provided an incomplete response in audio',
        group: 'audio-analysis',
        valueType: 'none',
        mandatory: false,
        placeholder: '',
        helpText: '',
        additionalFields: [],
        // allowObserve: 0,
    },
    {
        name: 'audio-stt-correctness',
        label: '',
        description: 'Detects STT correctness in audio',
        group: 'audio-analysis',
        valueType: 'none',
        mandatory: false,
        placeholder: '',
        helpText: '',
        additionalFields: [],
        // allowObserve: 0,
    },
    {
        name: 'audio-stt-missing',
        label: '',
        description: 'Detects STT missing in audio',
        group: 'audio-analysis',
        valueType: 'none',
        mandatory: false,
        placeholder: '',
        helpText: '',
        additionalFields: [],
        // allowObserve: 0,
    },
    // Video Analysis
    {
        name: 'video-visual-quality',
        label: '',
        description: 'Analyzes the quality of the video',
        group: 'video-analysis',
        valueType: 'none',
        mandatory: false,
        placeholder: '',
        helpText: '',
        additionalFields: [],
        // allowObserve: 0,
    },
    {
        name: 'text-to-video',
        label: '',
        description: 'Analyzes the alignment between the text prompt and the video content',
        group: 'video-analysis',
        valueType: 'none',
        mandatory: false,
        placeholder: '',
        helpText: '',
        additionalFields: [],
        // allowObserve: 0,
    },
    {
        name: 'video-temporal-consistency',
        label: '',
        description: 'Analyzes the consistency of objects or humans and the smoothness of motion or movements',
        group: 'video-analysis',
        valueType: 'none',
        mandatory: false,
        placeholder: '',
        helpText: '',
        additionalFields: [],
        // allowObserve: 0,
    },
    {
        name: 'video-factual-consistency',
        label: '',
        description: 'Analyzes the consistency of the video content with common-sense and factual knowledge',
        group: 'video-analysis',
        valueType: 'none',
        mandatory: false,
        placeholder: '',
        helpText: '',
        additionalFields: [],
        // allowObserve: 0,
    },
    {
        name: 'video-g-eval',
        label: 'Grading Criteria',
        description: 'Evaluates video outputs using G-Eval',
        group: 'video-analysis',
        valueType: 'array',
        placeholder: 'e.g., "The video is clear and of high quality."',
        helpText: 'Specify the criteria for the G-Eval evaluation',
        mandatory: true,
        additionalFields: [],
    },
    // Negated Assertions
    {
        name: 'not-equals',
        label: 'Expected Value',
        description: 'Checks if the output does not match the expected value',
        group: 'negated-assertions',
        valueType: 'string',
        placeholder: "e.g. 'error'",
        helpText: 'Enter the value that the output should not match.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'not-contains',
        label: 'Expected Value',
        description: 'Checks if the output does not contain the expected value',
        group: 'negated-assertions',
        valueType: 'string',
        placeholder: "e.g. 'failure'",
        helpText: 'Enter the text fragment that should not appear in the output.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'not-icontains',
        label: 'Expected Value',
        description: 'Case-insensitive check if the output does not contain the expected value',
        group: 'negated-assertions',
        valueType: 'string',
        placeholder: "e.g. 'error'",
        helpText: 'Enter the text fragment that should not appear in the output (case-insensitive).',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'not-contains-all',
        label: 'Expected Output',
        description: 'Checks if the output does not contain all of the expected values',
        group: 'negated-assertions',
        valueType: 'array',
        placeholder: "e.g. ['error', 'failed']",
        helpText: 'Enter text fragments that should not all appear together in the output.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'not-contains-any',
        label: 'Expected Output',
        description: 'Checks if the output does not contain any of the expected values',
        group: 'negated-assertions',
        valueType: 'array',
        placeholder: "e.g. ['error', 'invalid', 'failed']",
        helpText: 'Enter text fragments that should not appear in the output.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'not-starts-with',
        label: 'Expected Output not Starts with',
        description: 'Checks if the output does not start with the expected value',
        group: 'negated-assertions',
        valueType: 'string',
        placeholder: "e.g. 'Error:'",
        helpText: 'Enter the initial text the output should not begin with.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'not-regex',
        label: 'Expected Output not Regex',
        description: 'Checks if the output does not match the regular expression pattern',
        group: 'negated-assertions',
        valueType: 'regex',
        placeholder: "e.g. '^Error.*'",
        helpText: "Enter the regex pattern the output shouldn't match.",
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'not-is-json',
        label: '',
        description: 'Validates if the output is not valid JSON',
        group: 'negated-assertions',
        valueType: 'none',
        placeholder: "e.g. ['error', 'errorCode']",
        helpText: 'Enter JSON keys that should not be present or leave empty to check for invalid JSON format.',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'not-contains-json',
        label: '',
        description: 'Checks if the output does not contain valid JSON',
        group: 'negated-assertions',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'not-similar',
        label: 'Reference Text',
        description: 'Checks if output is not semantically similar to expected value',
        group: 'negated-assertions',
        valueType: 'array',
        placeholder: "e.g. 'The product has serious defects'",
        helpText: 'Enter a reference text to check for semantic non-similarity against the output.',
        mandatory: true,
        additionalFields: [],
    },
    // {
    //   name: 'not-webhook',
    //   label: '',
    //   description: 'Validates output does not pass external webhook service',
    //   group: 'negated-assertions',
    //   valueType: 'none',
    //   placeholder: '',
    //   helpText: '',
    //   mandatory: false,
    //   additionalFields: [],
    // },
    {
        name: 'not-rouge-n',
        label: '',
        description: 'Checks if ROUGE-N score is below threshold',
        group: 'negated-assertions',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    //Code Evaluation
    {
    /*
    name: 'code-execution',
    label: 'Expected Output',
    description:
      'Validates whether the provided code executes without errors and produces expected results',
    group: 'code-evaluation',
    valueType: 'string',
    placeholder: "e.g. 'Hello, world!'",
    helpText: 'Enter the expected output from code execution.',
    mandatory: true,
    additionalFields: [
      {
        name: 'language',
        type: 'single-select',
        label: 'Expected Language',
        placeholder: "e.g. 'Python'",
        helpText: 'Select the programming language of the code to execute.',
        description: 'Expected programming language in the code',
        options: ['javascript', 'python', 'java', 'c++', 'c#', 'go', 'rust', 'php', 'ruby'],
        mandatory: true,
      },
    ],
  */
    },
    {
        name: 'code-language',
        label: 'Expected Output Language',
        description: 'Identifies the programming language used in the code output',
        group: 'code-evaluation',
        valueType: 'single-select',
        options: ['javascript', 'python', 'java', 'c++', 'c#', 'go', 'rust', 'php', 'ruby'],
        placeholder: 'Select',
        helpText: 'Select the expected programming language of the code.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'contains-code-language',
        label: 'Expected Output Language',
        description: 'Checks if the output contains code written in the expected programming language',
        group: 'code-evaluation',
        valueType: 'single-select',
        options: ['javascript', 'python', 'java', 'c++', 'c#', 'go', 'rust', 'php', 'ruby'],
        placeholder: 'Select',
        helpText: 'Select the expected outout code language',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'code-readability',
        label: '',
        description: 'Evaluates how readable and understandable the code is (likely a typo in name: should be "code-readability")',
        group: 'code-evaluation',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'code-correctness',
        label: '',
        description: 'Checks if the code is logically and syntactically correct for the given problem',
        group: 'code-evaluation',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    // Converston Evaluation
    {
        name: 'conversational-g-eval',
        label: 'Evaluation Criteria',
        description: 'Evaluates the overall quality of a conversation using G-Eval',
        group: 'conversational-evaluation',
        valueType: 'string',
        placeholder: "e.g., 'The conversation should be professional'",
        helpText: 'Describe the evaluation aspects of the output',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'knowledge-retention',
        label: '',
        description: 'Measures how well the conversation retains and builds upon prior knowledge',
        group: 'conversational-evaluation',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'conversation-completeness',
        label: '',
        description: 'Checks if the conversation covers all expected topics or goals',
        group: 'conversational-evaluation',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'conversation-relevancy',
        label: '',
        description: 'Assesses how relevant the conversation is to the intended topic or context',
        group: 'conversational-evaluation',
        valueType: 'string',
        placeholder: '',
        helpText: '',
        mandatory: true,
        additionalFields: [
            {
                name: 'context',
                type: 'string',
                label: 'Context',
                helpText: 'Enter the context to check for faithfulness',
                mandatory: true,
                description: '',
                placeholder: '',
            },
        ],
    },
    {
        name: 'role-adherence',
        label: '',
        description: 'Evaluates how well the conversation stays aligned with the intended role, topic, or context.',
        group: 'conversational-evaluation',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    // Multimodal Assertions
    {
        name: 'text-to-image',
        label: '',
        description: 'Evaluates the quality of synthesized images based on semantic consistency and perceptual quality.',
        group: 'multimodal',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'image-helpfulness',
        label: '',
        description: 'Evaluates the helpfulness of an image in a multimodal context',
        group: 'multimodal',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'image-coherence',
        label: '',
        description: 'Assesses the coherent alignment of images with their accompanying text, evaluating how effectively the visual content complements and enhances the textual narrative.',
        group: 'multimodal',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'image-reference',
        label: '',
        description: 'Evaluates how accurately images are referred to or explained by accompanying text.',
        group: 'multimodal',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'multimodal-answer-relevancy',
        label: '',
        description: 'Evaluates how relevant the output of the multimodal application is compared to the provided input.',
        group: 'multimodal',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'multimodal-contextual-relevancy',
        label: 'Reference Output',
        description: 'Checks the contextual relevancy of multimodal content',
        group: 'multimodal',
        valueType: 'array',
        placeholder: 'e.g., ["A dog playing fetch in the park", "image_of_dog_fetching.png"]',
        helpText: 'Provide the expected reference content in both text and images. Evaluates how well the generated output aligns contextually with the reference',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'multimodal-contextual-recall',
        label: 'Reference Output',
        description: 'Checks the contextual recall of multimodal content',
        group: 'multimodal',
        valueType: 'array',
        placeholder: 'e.g., ["A red apple on a wooden table", "image_of_apple_on_table.jpg"]',
        helpText: 'Provide the reference content covering all essential elements in text and images. Evaluates how completely the output recalls the context of the reference.',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'multimodal-g-eval',
        label: 'Grading Criteria',
        description: 'Evaluates multi-modal outputs using G-Eval',
        group: 'multimodal',
        valueType: 'array',
        placeholder: 'e.g., "Evaluate the response for fluency, coherence, and relevance."',
        helpText: 'Specify the criteria for the G-Eval evaluation',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'task-completion',
        //label: 'Framework',
        label: '',
        //description: 'Choose the framework of the code',
        description: 'Checks if the task completion is correct',
        group: 'agent-metrics',
        //valueType: 'single-select',
        //options: ['langchain', 'langgraph', 'autogen', 'sdk'],
        valueType: 'none',
        placeholder: "e.g., 'Web Search'",
        helpText: 'Enter the expected task completion of the code',
        mandatory: true,
        additionalFields: [],
    },
    {
        name: 'tool-correctness',
        //label: 'Framework',
        label: '',
        //description: 'Choose the framework of the code',
        description: 'Checks if the tools used in the code are correct',
        group: 'agent-metrics',
        //valueType: 'single-select',
        //options: ['langchain', 'langgraph', 'autogen', 'sdk'],
        valueType: 'none',
        placeholder: "e.g., 'Web Search'",
        helpText: 'Enter the expected framework of the code',
        mandatory: true,
        additionalFields: [
            {
                name: 'tools',
                type: 'string',
                label: 'Expected Tools',
                placeholder: "e.g., 'Web Researcher'",
                helpText: 'Enter the expected tools of the code',
                description: 'Enter the expected tools of the code',
                mandatory: true,
            },
        ],
    },
    {
        name: 'argument-correctness',
        label: '',
        description: 'Assesses the AI application ability to generate the correct arguments for the tools it calls',
        group: 'agent-metrics',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'plan-quality',
        label: '',
        description: 'Evaluates whether an agent plan will accomplish the user task using only valid and applicable tools',
        group: 'agent-metrics',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
        allowObserve: 0,
    },
    {
        name: 'plan-adherence',
        label: '',
        description: 'Evaluates how well the agent adhered to its plan (compares actual tool_calls to the stated plan)',
        group: 'agent-metrics',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
        allowObserve: 0,
    },
    {
        name: 'goal-fulfillment',
        label: '',
        description: 'Evaluates whether the agent\'s plan and output fulfilled the user\'s original query',
        group: 'agent-metrics',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'execution-efficiency',
        label: '',
        description: 'Evaluates how efficiently the agent executed its steps (score 0–3 normalized to 0–1)',
        group: 'agent-metrics',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'reasoning-coherence',
        label: '',
        description: 'Evaluates logical consistency of the plan and execution (score 0–3 normalized to 0–1)',
        group: 'agent-metrics',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'agent-custom-eval',
        label: 'Grading Criteria',
        description: 'Evaluates agent plan and execution against custom criteria (array of instructions)',
        group: 'agent-metrics',
        valueType: 'array',
        placeholder: 'e.g., "If the user asks for personal information, the plan must include an authentication step."',
        helpText: 'Specify the criteria for the evaluation of the agent plan and execution',
        mandatory: true,
        additionalFields: [
            {
                name: 'threshold',
                type: 'number',
                label: 'Minimum evaluation Score',
                helpText: 'Enter the minimum score for the eval to pass',
                mandatory: false,
                description: 'Minimum score for the eval to pass',
                placeholder: '0.7',
                defaultValue: 0.7,
            },
        ],
    },
    {
        name: 'tool-selection-quality',
        label: '',
        description: 'Evaluates whether the LLM selected the most appropriate tool(s) for the task',
        group: 'agent-metrics',
        valueType: 'none',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    // External Service Assertions
    // {
    //   name: 'webhook',
    //   label: '',
    //   description: 'Validates output using external webhook service',
    //   group: 'external-services',
    //   valueType: 'none',
    //   placeholder: '',
    //   helpText: '',
    //   mandatory: false,
    //   additionalFields: [
    //     {
    //       name: 'webhookUrl',
    //       type: 'string',
    //       label: 'Webhook URL',
    //       description: 'URL for the webhook service',
    //       mandatory: true,
    //     },
    //   ],
    // },
    {
        name: 'is-valid-openai-function-call',
        description: 'Validates OpenAI function call format',
        group: 'external-services',
        valueType: 'string',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
    {
        name: 'is-valid-openai-tools-call',
        description: 'Validates OpenAI tools call format',
        group: 'external-services',
        valueType: 'string',
        placeholder: '',
        helpText: '',
        mandatory: false,
        additionalFields: [],
    },
];
async function seedAssertions() {
    const db = await (0, index_1.getDb)();
    // Delete all existing assertions first
    try {
        await db.delete(tables_1.assertionsTable).execute();
    }
    catch (error) {
        console.error('Error deleting assertions:', error);
    }
    try {
        for (const assertion of exports.assertionsSeed) {
            if (Object.keys(assertion).length === 0) {
                continue;
            }
            await db
                .insert(tables_1.assertionsTable)
                .values({
                id: (0, uuid_1.v4)(),
                name: assertion.name || '',
                label: assertion.label || null,
                description: assertion.description || null,
                placeholder: assertion.placeholder || null,
                helpText: assertion.helpText || null,
                allowEval: assertion?.allowEval === 0 ? 0 : 1,
                allowObserve: assertion?.allowObserve === 0 ? 0 : 1,
                group: assertion.group || '',
                valueType: assertion.valueType || '',
                mandatory: assertion.mandatory ? 1 : 0,
                additionalFields: assertion.additionalFields || null,
                options: assertion.options || null,
                providerResponseRequirements: assertion.providerResponseRequirements ??
                    (0, inferProviderResponseRequirements_1.inferProviderResponseRequirements)(assertion.name || '', assertion.group || ''),
            });
        }
        console.log('Assertions seeded successfully');
    }
    catch (error) {
        console.error('Error seeding assertions:', error);
    }
}
//# sourceMappingURL=assertions.js.map