"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const assertions_1 = require("../../src/assertions");
const openai_1 = require("../../src/assertions/openai");
const openai_2 = require("../../src/providers/openai");
jest.mock('fs');
jest.mock('path');
const mockedFs = jest.mocked(fs_1.default);
const mockedPath = jest.mocked(path_1.default);
const toolsAssertion = {
    type: 'is-valid-openai-tools-call',
};
const functionAssertion = {
    type: 'is-valid-openai-function-call',
};
const mockProvider = {
    id: () => 'test-provider',
    config: {
        tools: [
            {
                type: 'function',
                function: {
                    name: 'getCurrentTemperature',
                    parameters: {
                        type: 'object',
                        properties: {
                            location: { type: 'string' },
                            unit: { type: 'string', enum: ['Celsius', 'Fahrenheit'] },
                        },
                        required: ['location', 'unit'],
                    },
                },
            },
        ],
        functions: [
            {
                name: 'getCurrentTemperature',
                parameters: {
                    type: 'object',
                    properties: {
                        location: { type: 'string' },
                        unit: { type: 'string', enum: ['Celsius', 'Fahrenheit'] },
                    },
                    required: ['location', 'unit'],
                },
            },
        ],
    },
    callApi: async () => ({ output: '' }),
};
const mockContext = {
    prompt: '',
    vars: {},
    test: { vars: {} },
    logProbs: [],
    provider: mockProvider,
    providerResponse: { output: '' },
};
describe('OpenAI assertions', () => {
    beforeEach(() => {
        jest.resetAllMocks();
        mockedPath.resolve.mockImplementation((...args) => args[args.length - 1]);
        mockedFs.existsSync.mockReturnValue(true);
    });
    describe('is-valid-openai-function-call assertion', () => {
        it('should pass for a valid function call with correct arguments', async () => {
            const output = { arguments: '{"x": 10, "y": 20}', name: 'add' };
            const provider = new openai_2.OpenAiChatCompletionProvider('foo', {
                config: {
                    functions: [
                        {
                            name: 'add',
                            parameters: {
                                type: 'object',
                                properties: {
                                    x: { type: 'number' },
                                    y: { type: 'number' },
                                },
                                required: ['x', 'y'],
                            },
                        },
                    ],
                },
            });
            const providerResponse = { output };
            const result = await (0, assertions_1.runAssertion)({
                prompt: 'Some prompt',
                provider,
                assertion: {
                    type: 'is-valid-openai-function-call',
                },
                test: {},
                providerResponse,
            });
            expect(result).toMatchObject({
                pass: true,
                reason: 'Assertion passed',
            });
        });
        it('should pass for a nested function_call output', async () => {
            const output = {
                function_call: { arguments: '{"x": 10, "y": 20}', name: 'add' },
            };
            const provider = new openai_2.OpenAiChatCompletionProvider('foo', {
                config: {
                    functions: [
                        {
                            name: 'add',
                            parameters: {
                                type: 'object',
                                properties: {
                                    x: { type: 'number' },
                                    y: { type: 'number' },
                                },
                                required: ['x', 'y'],
                            },
                        },
                    ],
                },
            });
            const providerResponse = { output };
            const result = await (0, assertions_1.runAssertion)({
                prompt: 'Some prompt',
                provider,
                assertion: {
                    type: 'is-valid-openai-function-call',
                },
                test: {},
                providerResponse,
            });
            expect(result).toMatchObject({
                pass: true,
                reason: 'Assertion passed',
            });
        });
        it('should fail for an invalid function call with incorrect arguments', async () => {
            const output = { arguments: '{"x": "10", "y": 20}', name: 'add' };
            const result = await (0, assertions_1.runAssertion)({
                prompt: 'Some prompt',
                provider: new openai_2.OpenAiChatCompletionProvider('foo', {
                    config: {
                        functions: [
                            {
                                name: 'add',
                                parameters: {
                                    type: 'object',
                                    properties: {
                                        x: { type: 'number' },
                                        y: { type: 'number' },
                                    },
                                    required: ['x', 'y'],
                                },
                            },
                        ],
                    },
                }),
                assertion: {
                    type: 'is-valid-openai-function-call',
                },
                test: {},
                providerResponse: { output },
            });
            expect(result).toMatchObject({
                pass: false,
                reason: expect.stringContaining('Call to "add" does not match schema'),
            });
        });
    });
    describe('handleIsValidOpenAiFunctionCall', () => {
        it('should pass when function call matches schema', () => {
            const functionOutput = {
                name: 'getCurrentTemperature',
                arguments: '{"location": "San Francisco, CA", "unit": "Fahrenheit"}',
            };
            const result = (0, openai_1.handleIsValidOpenAiFunctionCall)({
                assertion: functionAssertion,
                output: functionOutput,
                provider: mockProvider,
                test: { vars: {} },
                baseType: functionAssertion.type,
                context: mockContext,
                inverse: false,
                outputString: JSON.stringify(functionOutput),
                providerResponse: { output: functionOutput },
            });
            expect(result).toEqual({
                pass: true,
                score: 1,
                reason: 'Assertion passed',
                assertion: functionAssertion,
            });
        });
        it('should load functions from external file', () => {
            const functionOutput = {
                name: 'getCurrentTemperature',
                arguments: '{"location": "San Francisco, CA", "unit": "Fahrenheit"}',
            };
            const mockYamlContent = `
- name: getCurrentTemperature
  parameters:
    type: object
    properties:
      location:
        type: string
      unit:
        type: string
        enum: [Celsius, Fahrenheit]
    required: [location, unit]
`;
            mockedFs.readFileSync.mockReturnValue(mockYamlContent);
            const fileProvider = {
                ...mockProvider,
                config: {
                    functions: 'file://./test/fixtures/weather_functions.yaml',
                },
            };
            const result = (0, openai_1.handleIsValidOpenAiFunctionCall)({
                assertion: functionAssertion,
                output: functionOutput,
                provider: fileProvider,
                test: { vars: {} },
                baseType: functionAssertion.type,
                context: mockContext,
                inverse: false,
                outputString: JSON.stringify(functionOutput),
                providerResponse: { output: functionOutput },
            });
            expect(result).toEqual({
                pass: true,
                score: 1,
                reason: 'Assertion passed',
                assertion: functionAssertion,
            });
            expect(mockedFs.existsSync).toHaveBeenCalledWith('./test/fixtures/weather_functions.yaml');
            expect(mockedFs.readFileSync).toHaveBeenCalledWith('./test/fixtures/weather_functions.yaml', 'utf8');
        });
        it('should render variables in function definitions', () => {
            const functionOutput = {
                name: 'getCurrentTemperature',
                arguments: '{"location": "San Francisco, CA", "unit": "custom_unit"}',
            };
            const varProvider = {
                ...mockProvider,
                config: {
                    functions: [
                        {
                            name: 'getCurrentTemperature',
                            parameters: {
                                type: 'object',
                                properties: {
                                    location: { type: 'string' },
                                    unit: { type: 'string', enum: ['{{unit}}'] },
                                },
                                required: ['location', 'unit'],
                            },
                        },
                    ],
                },
            };
            const result = (0, openai_1.handleIsValidOpenAiFunctionCall)({
                assertion: functionAssertion,
                output: functionOutput,
                provider: varProvider,
                test: { vars: { unit: 'custom_unit' } },
                baseType: functionAssertion.type,
                context: mockContext,
                inverse: false,
                outputString: JSON.stringify(functionOutput),
                providerResponse: { output: functionOutput },
            });
            expect(result).toEqual({
                pass: true,
                score: 1,
                reason: 'Assertion passed',
                assertion: functionAssertion,
            });
        });
        it('should fail when functions are not defined', () => {
            const functionOutput = {
                name: 'getCurrentTemperature',
                arguments: '{"location": "San Francisco, CA"}',
            };
            const emptyProvider = {
                ...mockProvider,
                config: {},
            };
            const result = (0, openai_1.handleIsValidOpenAiFunctionCall)({
                assertion: functionAssertion,
                output: functionOutput,
                provider: emptyProvider,
                test: { vars: {} },
                baseType: functionAssertion.type,
                context: mockContext,
                inverse: false,
                outputString: JSON.stringify(functionOutput),
                providerResponse: { output: functionOutput },
            });
            expect(result).toEqual({
                pass: false,
                score: 0,
                reason: 'Called "getCurrentTemperature", but there is no function with that name',
                assertion: functionAssertion,
            });
        });
        it('should fail when function output is not an object', () => {
            const functionOutput = 'not an object';
            const result = (0, openai_1.handleIsValidOpenAiFunctionCall)({
                assertion: functionAssertion,
                output: functionOutput,
                provider: mockProvider,
                test: { vars: {} },
                baseType: functionAssertion.type,
                context: mockContext,
                inverse: false,
                outputString: JSON.stringify(functionOutput),
                providerResponse: { output: functionOutput },
            });
            expect(result).toEqual({
                pass: false,
                score: 0,
                reason: expect.stringContaining('OpenAI did not return a valid-looking function call'),
                assertion: functionAssertion,
            });
        });
        it('should fail when function call does not match schema', () => {
            const functionOutput = {
                name: 'getCurrentTemperature',
                arguments: '{"location": "San Francisco, CA"}', // missing required 'unit'
            };
            const result = (0, openai_1.handleIsValidOpenAiFunctionCall)({
                assertion: functionAssertion,
                output: functionOutput,
                provider: mockProvider,
                test: { vars: {} },
                baseType: functionAssertion.type,
                context: mockContext,
                inverse: false,
                outputString: JSON.stringify(functionOutput),
                providerResponse: { output: functionOutput },
            });
            expect(result).toEqual({
                pass: false,
                score: 0,
                reason: expect.stringContaining('must have required property'),
                assertion: functionAssertion,
            });
        });
    });
    describe('is-valid-openai-tools-call assertion', () => {
        it('should pass for a valid tools call with correct arguments', async () => {
            const output = [
                { type: 'function', function: { arguments: '{"x": 10, "y": 20}', name: 'add' } },
            ];
            const result = await (0, assertions_1.runAssertion)({
                prompt: 'Some prompt',
                provider: new openai_2.OpenAiChatCompletionProvider('foo', {
                    config: {
                        tools: [
                            {
                                type: 'function',
                                function: {
                                    name: 'add',
                                    parameters: {
                                        type: 'object',
                                        properties: {
                                            x: { type: 'number' },
                                            y: { type: 'number' },
                                        },
                                        required: ['x', 'y'],
                                    },
                                },
                            },
                        ],
                    },
                }),
                assertion: {
                    type: 'is-valid-openai-tools-call',
                },
                test: {},
                providerResponse: { output },
            });
            expect(result).toMatchObject({
                pass: true,
                reason: 'Assertion passed',
            });
        });
        it('should pass for a nested tool_calls output', async () => {
            const output = {
                content: 'Let me check the weather...',
                tool_calls: [
                    {
                        id: 'call_123',
                        type: 'function',
                        function: {
                            name: 'getCurrentTemperature',
                            arguments: '{"location": "San Francisco", "unit": "Fahrenheit"}',
                        },
                    },
                ],
            };
            const result = await (0, assertions_1.runAssertion)({
                prompt: 'Some prompt',
                provider: new openai_2.OpenAiChatCompletionProvider('foo', {
                    config: {
                        tools: [
                            {
                                type: 'function',
                                function: {
                                    name: 'getCurrentTemperature',
                                    parameters: {
                                        type: 'object',
                                        properties: {
                                            location: { type: 'string' },
                                            unit: { type: 'string', enum: ['Celsius', 'Fahrenheit'] },
                                        },
                                        required: ['location', 'unit'],
                                    },
                                },
                            },
                        ],
                    },
                }),
                assertion: {
                    type: 'is-valid-openai-tools-call',
                },
                test: {},
                providerResponse: { output },
            });
            expect(result).toMatchObject({
                pass: true,
                reason: 'Assertion passed',
            });
        });
        it('should pass for multiple tool calls in parallel', async () => {
            const output = {
                content: 'Checking weather data...',
                tool_calls: [
                    {
                        id: 'call_123',
                        type: 'function',
                        function: {
                            name: 'getCurrentTemperature',
                            arguments: '{"location": "London", "unit": "Celsius"}',
                        },
                    },
                    {
                        id: 'call_456',
                        type: 'function',
                        function: {
                            name: 'getRainProbability',
                            arguments: '{"location": "London"}',
                        },
                    },
                ],
            };
            const result = await (0, assertions_1.runAssertion)({
                prompt: 'Some prompt',
                provider: new openai_2.OpenAiChatCompletionProvider('foo', {
                    config: {
                        tools: [
                            {
                                type: 'function',
                                function: {
                                    name: 'getCurrentTemperature',
                                    parameters: {
                                        type: 'object',
                                        properties: {
                                            location: { type: 'string' },
                                            unit: { type: 'string', enum: ['Celsius', 'Fahrenheit'] },
                                        },
                                        required: ['location', 'unit'],
                                    },
                                },
                            },
                            {
                                type: 'function',
                                function: {
                                    name: 'getRainProbability',
                                    parameters: {
                                        type: 'object',
                                        properties: {
                                            location: { type: 'string' },
                                        },
                                        required: ['location'],
                                    },
                                },
                            },
                        ],
                    },
                }),
                assertion: {
                    type: 'is-valid-openai-tools-call',
                },
                test: {},
                providerResponse: { output },
            });
            expect(result).toMatchObject({
                pass: true,
                reason: 'Assertion passed',
            });
        });
        it('should fail for an invalid tools call with incorrect arguments', async () => {
            const output = [
                { type: 'function', function: { arguments: '{"x": "foobar", "y": 20}', name: 'add' } },
            ];
            const result = await (0, assertions_1.runAssertion)({
                prompt: 'Some prompt',
                provider: new openai_2.OpenAiChatCompletionProvider('foo', {
                    config: {
                        tools: [
                            {
                                type: 'function',
                                function: {
                                    name: 'add',
                                    parameters: {
                                        type: 'object',
                                        properties: {
                                            x: { type: 'number' },
                                            y: { type: 'number' },
                                        },
                                        required: ['x', 'y'],
                                    },
                                },
                            },
                        ],
                    },
                }),
                assertion: {
                    type: 'is-valid-openai-tools-call',
                },
                test: {},
                providerResponse: { output },
            });
            expect(result).toMatchObject({
                pass: false,
                reason: expect.stringContaining('Call to "add" does not match schema'),
            });
        });
    });
    describe('handleIsValidOpenAiToolsCall', () => {
        // Basic validation tests
        it('should pass when tool calls match schema', () => {
            const toolsOutput = [
                {
                    id: 'call_123',
                    type: 'function',
                    function: {
                        name: 'getCurrentTemperature',
                        arguments: '{"location": "San Francisco, CA", "unit": "Fahrenheit"}',
                    },
                },
            ];
            const result = (0, openai_1.handleIsValidOpenAiToolsCall)({
                assertion: toolsAssertion,
                output: toolsOutput,
                provider: mockProvider,
                test: { vars: {} },
                baseType: toolsAssertion.type,
                context: mockContext,
                inverse: false,
                outputString: JSON.stringify(toolsOutput),
                providerResponse: { output: toolsOutput },
            });
            expect(result).toEqual({
                pass: true,
                score: 1,
                reason: 'Assertion passed',
                assertion: toolsAssertion,
            });
        });
        // External file loading tests
        it('should load tools from external file', () => {
            const toolsOutput = [
                {
                    id: 'call_123',
                    type: 'function',
                    function: {
                        name: 'getCurrentTemperature',
                        arguments: '{"location": "San Francisco, CA", "unit": "Fahrenheit"}',
                    },
                },
            ];
            const mockYamlContent = `
- type: function
  function:
    name: getCurrentTemperature
    parameters:
      type: object
      properties:
        location:
          type: string
        unit:
          type: string
          enum: [Celsius, Fahrenheit]
      required: [location, unit]
`;
            mockedFs.readFileSync.mockReturnValue(mockYamlContent);
            const fileProvider = {
                ...mockProvider,
                config: {
                    tools: 'file://./test/fixtures/weather_tools.yaml',
                },
            };
            const result = (0, openai_1.handleIsValidOpenAiToolsCall)({
                assertion: toolsAssertion,
                output: toolsOutput,
                provider: fileProvider,
                test: { vars: {} },
                baseType: toolsAssertion.type,
                context: mockContext,
                inverse: false,
                outputString: JSON.stringify(toolsOutput),
                providerResponse: { output: toolsOutput },
            });
            expect(result).toEqual({
                pass: true,
                score: 1,
                reason: 'Assertion passed',
                assertion: toolsAssertion,
            });
            expect(mockedFs.existsSync).toHaveBeenCalledWith('./test/fixtures/weather_tools.yaml');
            expect(mockedFs.readFileSync).toHaveBeenCalledWith('./test/fixtures/weather_tools.yaml', 'utf8');
        });
        // Variable rendering tests
        it('should render variables in tool definitions', () => {
            const toolsOutput = [
                {
                    id: 'call_123',
                    type: 'function',
                    function: {
                        name: 'getCurrentTemperature',
                        arguments: '{"location": "San Francisco, CA", "unit": "custom_unit"}',
                    },
                },
            ];
            const varProvider = {
                ...mockProvider,
                config: {
                    tools: [
                        {
                            type: 'function',
                            function: {
                                name: 'getCurrentTemperature',
                                parameters: {
                                    type: 'object',
                                    properties: {
                                        location: { type: 'string' },
                                        unit: { type: 'string', enum: ['{{unit}}'] },
                                    },
                                    required: ['location', 'unit'],
                                },
                            },
                        },
                    ],
                },
            };
            const result = (0, openai_1.handleIsValidOpenAiToolsCall)({
                assertion: toolsAssertion,
                output: toolsOutput,
                provider: varProvider,
                test: { vars: { unit: 'custom_unit' } },
                baseType: toolsAssertion.type,
                context: mockContext,
                inverse: false,
                outputString: JSON.stringify(toolsOutput),
                providerResponse: { output: toolsOutput },
            });
            expect(result).toEqual({
                pass: true,
                score: 1,
                reason: 'Assertion passed',
                assertion: toolsAssertion,
            });
        });
        // Error cases
        it('should fail when tools are not defined', () => {
            const toolsOutput = [
                {
                    id: 'call_123',
                    type: 'function',
                    function: {
                        name: 'getCurrentTemperature',
                        arguments: '{"location": "San Francisco, CA"}',
                    },
                },
            ];
            const emptyProvider = {
                ...mockProvider,
                config: {},
            };
            expect(() => (0, openai_1.handleIsValidOpenAiToolsCall)({
                assertion: toolsAssertion,
                output: toolsOutput,
                provider: emptyProvider,
                test: { vars: {} },
                baseType: toolsAssertion.type,
                context: mockContext,
                inverse: false,
                outputString: JSON.stringify(toolsOutput),
                providerResponse: { output: toolsOutput },
            })).toThrow('Tools are expected to be an array of objects with a function property');
        });
        it('should fail when tool output is not an array', () => {
            const toolsOutput = {
                id: 'call_123',
                type: 'function',
                function: {
                    name: 'getCurrentTemperature',
                    arguments: '{"location": "San Francisco, CA"}',
                },
            };
            const emptyToolsProvider = {
                ...mockProvider,
                config: {
                    tools: [],
                },
            };
            const result = (0, openai_1.handleIsValidOpenAiToolsCall)({
                assertion: toolsAssertion,
                output: toolsOutput,
                provider: emptyToolsProvider,
                test: { vars: {} },
                baseType: toolsAssertion.type,
                context: mockContext,
                inverse: false,
                outputString: JSON.stringify(toolsOutput),
                providerResponse: { output: toolsOutput },
            });
            expect(result).toEqual({
                pass: false,
                score: 0,
                reason: expect.stringContaining('OpenAI did not return a valid-looking tools response'),
                assertion: toolsAssertion,
            });
        });
        it('should fail when tool call does not match schema', () => {
            const toolsOutput = [
                {
                    id: 'call_123',
                    type: 'function',
                    function: {
                        name: 'getCurrentTemperature',
                        arguments: '{"location": "San Francisco, CA"}', // missing required 'unit'
                    },
                },
            ];
            const result = (0, openai_1.handleIsValidOpenAiToolsCall)({
                assertion: toolsAssertion,
                output: toolsOutput,
                provider: mockProvider,
                test: { vars: {} },
                baseType: toolsAssertion.type,
                context: mockContext,
                inverse: false,
                outputString: JSON.stringify(toolsOutput),
                providerResponse: { output: toolsOutput },
            });
            expect(result).toEqual({
                pass: false,
                score: 0,
                reason: expect.stringContaining('must have required property'),
                assertion: toolsAssertion,
            });
        });
    });
});
//# sourceMappingURL=openai.test.js.map