"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.readVarsFiles = readVarsFiles;
exports.readStandaloneTestsFile = readStandaloneTestsFile;
exports.readTest = readTest;
exports.readTests = readTests;
exports.generatePersonasPrompt = generatePersonasPrompt;
exports.testCasesPrompt = testCasesPrompt;
exports.synthesize = synthesize;
exports.synthesizeFromTestSuite = synthesizeFromTestSuite;
const json_schema_ref_parser_1 = __importDefault(require("@apidevtools/json-schema-ref-parser"));
const sync_1 = require("csv-parse/sync");
const dedent_1 = __importDefault(require("dedent"));
const fs = __importStar(require("fs"));
const glob_1 = require("glob");
const js_yaml_1 = __importDefault(require("js-yaml"));
const path = __importStar(require("path"));
const path_1 = require("path");
const csv_1 = require("./csv");
const envars_1 = require("./envars");
const esm_1 = require("./esm");
const googleSheets_1 = require("./googleSheets");
const huggingfaceDatasets_1 = require("./integrations/huggingfaceDatasets");
const logger_1 = __importDefault(require("./logger"));
const providers_1 = require("./providers");
const defaults_1 = require("./providers/defaults");
const telemetry_1 = __importDefault(require("./telemetry"));
const generation_1 = require("./util/generation");
const invariant_1 = __importDefault(require("./util/invariant"));
const json_1 = require("./util/json");
const templates_1 = require("./util/templates");
function parseJson(json) {
    try {
        return JSON.parse(json);
    }
    catch {
        return undefined;
    }
}
async function readVarsFiles(pathOrGlobs, basePath = '') {
    if (typeof pathOrGlobs === 'string') {
        pathOrGlobs = [pathOrGlobs];
    }
    const ret = {};
    for (const pathOrGlob of pathOrGlobs) {
        const resolvedPath = path.resolve(basePath, pathOrGlob);
        const paths = (0, glob_1.globSync)(resolvedPath, {
            windowsPathsNoEscape: true,
        });
        for (const p of paths) {
            const yamlData = js_yaml_1.default.load(fs.readFileSync(p, 'utf-8'));
            Object.assign(ret, yamlData);
        }
    }
    return ret;
}
async function readStandaloneTestsFile(varsPath, basePath = '') {
    // This function is confusingly named - it reads a CSV, JSON, or YAML file of
    // TESTS or test equivalents.
    const resolvedVarsPath = path.resolve(basePath, varsPath.replace(/^file:\/\//, ''));
    const fileExtension = (0, path_1.parse)(resolvedVarsPath).ext.slice(1);
    let rows = [];
    if (varsPath.startsWith('https://docs.google.com/spreadsheets/')) {
        telemetry_1.default.recordAndSendOnce('feature_used', {
            feature: 'csv tests file - google sheet',
        });
        rows = await (0, googleSheets_1.fetchCsvFromGoogleSheet)(varsPath);
    }
    else if (fileExtension === 'csv') {
        telemetry_1.default.recordAndSendOnce('feature_used', {
            feature: 'csv tests file - local',
        });
        const delimiter = (0, envars_1.getEnvString)('PROMPTFOO_CSV_DELIMITER', ',');
        rows = (0, sync_1.parse)(fs.readFileSync(resolvedVarsPath, 'utf-8'), {
            columns: true,
            bom: true,
            delimiter,
        });
    }
    else if (fileExtension === 'json') {
        telemetry_1.default.recordAndSendOnce('feature_used', {
            feature: 'json tests file',
        });
        rows = parseJson(fs.readFileSync(resolvedVarsPath, 'utf-8'));
    }
    else if (fileExtension === 'yaml' || fileExtension === 'yml') {
        telemetry_1.default.recordAndSendOnce('feature_used', {
            feature: 'yaml tests file',
        });
        rows = js_yaml_1.default.load(fs.readFileSync(resolvedVarsPath, 'utf-8'));
    }
    else if (fileExtension === 'js' || fileExtension === 'ts' || fileExtension === 'mjs') {
        telemetry_1.default.recordAndSendOnce('feature_used', {
            feature: 'js tests file',
        });
        return await (0, esm_1.importModule)(resolvedVarsPath);
    }
    return rows.map((row, idx) => {
        const test = (0, csv_1.testCaseFromCsvRow)(row);
        test.description || (test.description = `Row #${idx + 1}`);
        return test;
    });
}
async function loadTestWithVars(testCase, testBasePath) {
    const ret = { ...testCase, vars: undefined };
    if (typeof testCase.vars === 'string' || Array.isArray(testCase.vars)) {
        ret.vars = await readVarsFiles(testCase.vars, testBasePath);
    }
    else {
        ret.vars = testCase.vars;
    }
    return ret;
}
async function readTest(test, basePath = '') {
    let testCase;
    if (typeof test === 'string') {
        const testFilePath = path.resolve(basePath, test);
        const testBasePath = path.dirname(testFilePath);
        const rawTestCase = js_yaml_1.default.load(fs.readFileSync(testFilePath, 'utf-8'));
        testCase = await loadTestWithVars(rawTestCase, testBasePath);
    }
    else {
        testCase = await loadTestWithVars(test, basePath);
    }
    if (testCase.provider && typeof testCase.provider !== 'function') {
        // Load provider
        if (typeof testCase.provider === 'string') {
            testCase.provider = await (0, providers_1.loadApiProvider)(testCase.provider);
        }
        else if (typeof testCase.provider.id === 'string') {
            testCase.provider = await (0, providers_1.loadApiProvider)(testCase.provider.id, {
                options: testCase.provider,
                basePath,
            });
        }
    }
    if (!testCase.assert &&
        !testCase.vars &&
        !testCase.options &&
        !testCase.metadata &&
        !testCase.provider &&
        !testCase.providerOutput &&
        typeof testCase.threshold !== 'number') {
        // Validate the shape of the test case
        // We skip validation when loading the default test case, since it may not have all the properties
        throw new Error(`Test case must contain one of the following properties: assert, vars, options, metadata, provider, providerOutput, threshold.\n\nInstead got:\n${JSON.stringify(testCase, null, 2)}`);
    }
    return testCase;
}
async function readTests(tests, basePath = '') {
    const ret = [];
    const loadTestsFromGlob = async (loadTestsGlob) => {
        if (loadTestsGlob.startsWith('huggingface://datasets/')) {
            telemetry_1.default.recordAndSendOnce('feature_used', {
                feature: 'huggingface dataset',
            });
            return await (0, huggingfaceDatasets_1.fetchHuggingFaceDataset)(loadTestsGlob);
        }
        if (loadTestsGlob.startsWith('file://')) {
            loadTestsGlob = loadTestsGlob.slice('file://'.length);
        }
        const resolvedPath = path.resolve(basePath, loadTestsGlob);
        const testFiles = (0, glob_1.globSync)(resolvedPath, {
            windowsPathsNoEscape: true,
        });
        if (loadTestsGlob.startsWith('https://docs.google.com/spreadsheets/')) {
            testFiles.push(loadTestsGlob);
        }
        const _deref = async (testCases, file) => {
            logger_1.default.debug(`Dereferencing test file: ${file}`);
            return (await json_schema_ref_parser_1.default.dereference(testCases));
        };
        const ret = [];
        if (testFiles.length < 1) {
            logger_1.default.error(`No test files found for path: ${loadTestsGlob}`);
            return ret;
        }
        for (const testFile of testFiles) {
            let testCases;
            if (testFile.endsWith('.csv') ||
                testFile.startsWith('https://docs.google.com/spreadsheets/')) {
                testCases = await readStandaloneTestsFile(testFile, basePath);
            }
            else if (testFile.endsWith('.yaml') || testFile.endsWith('.yml')) {
                testCases = js_yaml_1.default.load(fs.readFileSync(testFile, 'utf-8'));
                testCases = await _deref(testCases, testFile);
            }
            else if (testFile.endsWith('.json')) {
                // eslint-disable-next-line @typescript-eslint/no-require-imports
                testCases = await _deref(require(testFile), testFile);
            }
            else {
                throw new Error(`Unsupported file type for test file: ${testFile}`);
            }
            if (testCases) {
                if (!Array.isArray(testCases) && typeof testCases === 'object') {
                    testCases = [testCases];
                }
                for (const testCase of testCases) {
                    ret.push(await readTest(testCase, path.dirname(testFile)));
                }
            }
        }
        return ret;
    };
    if (typeof tests === 'string') {
        if (tests.endsWith('yaml') || tests.endsWith('yml')) {
            // Points to a tests file with multiple test cases
            return loadTestsFromGlob(tests);
        }
        else {
            // Points to a tests.{csv,json,yaml,yml,js} or Google Sheet
            return readStandaloneTestsFile(tests, basePath);
        }
    }
    else if (Array.isArray(tests)) {
        for (const globOrTest of tests) {
            if (typeof globOrTest === 'string') {
                // Resolve globs
                ret.push(...(await loadTestsFromGlob(globOrTest)));
            }
            else {
                // Load individual TestCase
                ret.push(await readTest(globOrTest, basePath));
            }
        }
    }
    else if (tests !== undefined && tests !== null) {
        logger_1.default.warn((0, dedent_1.default) `
      Warning: Unsupported 'tests' format in promptfooconfig.yaml.
      Expected: string, string[], or TestCase[], but received: ${typeof tests}

      Please check your configuration file and ensure the 'tests' field is correctly formatted.
      For more information, visit: https://promptfoo.dev/docs/configuration/reference/#test-case
    `);
    }
    if (ret.some((testCase) => testCase.vars?.assert) &&
        !(0, envars_1.getEnvBool)('PROMPTFOO_NO_TESTCASE_ASSERT_WARNING')) {
        logger_1.default.warn((0, dedent_1.default) `
      Warning: Found 'assert' key in vars. This is likely a mistake in your configuration.

      'assert' should be *unindented* so it is under the test itself, not vars. For example:

      tests:
        - vars:
            foo: bar
          assert:
            - type: contains
              value: "bar"

      To disable this message, set the environment variable PROMPTFOO_NO_TESTCASE_ASSERT_WARNING=1.
    `);
    }
    return ret;
}
function generatePersonasPrompt(prompts, numPersonas) {
    const promptsString = (0, dedent_1.default) `<Prompts>
    ${prompts.map((prompt) => `<Prompt>\n${prompt}\n</Prompt>`).join('\n')}
    </Prompts>`;
    return (0, dedent_1.default) `
    Consider the following prompt${prompts.length > 1 ? 's' : ''} for an LLM application:

    ${promptsString}

    List up to ${numPersonas} user personas that would send ${prompts.length > 1 ? 'these prompts' : 'this prompt'}. Your response should be JSON of the form {personas: string[]}`;
}
function testCasesPrompt(prompts, persona, tests, numTestCasesPerPersona, variables, instructions) {
    const promptsString = (0, dedent_1.default) `
    <Prompts>
    ${prompts
        .map((prompt) => (0, dedent_1.default) `
      <Prompt>
      ${prompt}
      </Prompt>`)
        .join('\n')}
    </Prompts>`;
    const existingTests = (0, dedent_1.default) `
    Here are some existing tests:
    ${(0, generation_1.sampleArray)(tests, 100)
        .map((test) => {
        if (!test.vars) {
            return;
        }
        return (0, dedent_1.default) `
          <Test>
          ${JSON.stringify(test.vars, null, 2)}
          </Test>`;
    })
        .filter(Boolean)
        .sort()
        .join('\n')}
  `;
    return (0, dedent_1.default) `
    Consider ${prompts.length > 1 ? 'these prompts' : 'this prompt'}, which contains some {{variables}}:
  ${promptsString}

  This is your persona:
  <Persona>
  ${persona}
  </Persona>

  ${existingTests}

  Fully embody this persona and determine a value for each variable, such that the prompt would be sent by this persona.

  You are a tester, so try to think of ${numTestCasesPerPersona} sets of values that would be interesting or unusual to test.${instructions ? ` ${instructions}` : ''}

  Your response should contain a JSON map of variable names to values, of the form {vars: {${Array.from(variables)
        .map((varName) => `${varName}: string`)
        .join(', ')}}[]}`;
}
async function synthesize({ prompts, instructions, tests, numPersonas, numTestCasesPerPersona, provider, }) {
    if (prompts.length < 1) {
        throw new Error('Dataset synthesis requires at least one prompt.');
    }
    numPersonas = numPersonas || 5;
    numTestCasesPerPersona = numTestCasesPerPersona || 3;
    let progressBar;
    if (logger_1.default.level !== 'debug') {
        const cliProgress = await Promise.resolve().then(() => __importStar(require('cli-progress')));
        progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
        const totalProgressSteps = 1 + numPersonas * numTestCasesPerPersona;
        progressBar.start(totalProgressSteps, 0);
    }
    logger_1.default.debug(`Starting dataset synthesis. We'll begin by generating up to ${numPersonas} personas. Each persona will be used to generate ${numTestCasesPerPersona} test cases.`);
    logger_1.default.debug(`Generating user personas from ${prompts.length} prompt${prompts.length > 1 ? 's' : ''}...`);
    let providerModel;
    if (typeof provider === 'undefined') {
        providerModel = (await (0, defaults_1.getDefaultProviders)()).synthesizeProvider;
    }
    else {
        providerModel = await (0, providers_1.loadApiProvider)(provider);
    }
    const personasPrompt = generatePersonasPrompt(prompts, numPersonas);
    logger_1.default.debug(`Generated personas prompt:\n${personasPrompt}`);
    const resp = await providerModel.callApi(personasPrompt);
    logger_1.default.debug(`Received personas response:\n${resp.output}`);
    (0, invariant_1.default)(typeof resp.output !== 'undefined', 'resp.output must be defined');
    const output = typeof resp.output === 'string' ? resp.output : JSON.stringify(resp.output);
    const respObjects = (0, json_1.extractJsonObjects)(output);
    (0, invariant_1.default)(respObjects.length === 1, `Expected exactly one JSON object in the response for personas, got ${respObjects.length}`);
    const personas = respObjects[0].personas;
    logger_1.default.debug(`Generated ${personas.length} persona${personas.length === 1 ? '' : 's'}:\n${personas.map((p) => `  - ${p}`).join('\n')}`);
    if (progressBar) {
        progressBar.increment();
    }
    // Extract variable names from the nunjucks template in the prompts
    const variables = (0, templates_1.extractVariablesFromTemplates)(prompts);
    logger_1.default.debug(`Extracted ${variables.length} variable${variables.length === 1 ? '' : 's'} from prompt${prompts.length === 1 ? '' : 's'}:\n${variables
        .map((v) => `  - ${v}`)
        .join('\n')}`);
    const batchSize = 20;
    const totalTestCases = numPersonas * numTestCasesPerPersona;
    const generateTestCasesForPersona = async (currentTestCases) => {
        const remainingCount = totalTestCases - currentTestCases.length;
        const currentBatchSize = Math.min(remainingCount, batchSize);
        const persona = personas[currentTestCases.length % personas.length];
        logger_1.default.debug(`Generating ${currentBatchSize} test cases for persona ${(currentTestCases.length % personas.length) + 1} of ${personas.length}...`);
        const personaPrompt = testCasesPrompt(prompts, persona, tests, currentBatchSize, variables, instructions);
        logger_1.default.debug(`Generated persona prompt:\n${personaPrompt}`);
        const personaResponse = await providerModel.callApi(personaPrompt);
        logger_1.default.debug(`Received persona response:\n${personaResponse.output}`);
        const personaResponseObjects = (0, json_1.extractJsonObjects)(personaResponse.output);
        (0, invariant_1.default)(personaResponseObjects.length === 1, `Expected exactly one JSON object in the response for persona ${persona}, got ${personaResponseObjects.length}`);
        const parsed = personaResponseObjects[0];
        logger_1.default.debug(`Received ${parsed.vars.length} test cases`);
        if (progressBar) {
            progressBar.increment(parsed.vars.length);
        }
        return parsed.vars || [];
    };
    let testCaseVars = await (0, generation_1.retryWithDeduplication)(generateTestCasesForPersona, totalTestCases);
    logger_1.default.debug(`Generated ${testCaseVars.length} test cases`);
    if (testCaseVars.length > totalTestCases) {
        logger_1.default.debug(`Generated ${testCaseVars.length} test cases, but only ${totalTestCases} were requested. Sampling down to ${totalTestCases}...`);
        testCaseVars = (0, generation_1.sampleArray)(testCaseVars, totalTestCases);
    }
    if (progressBar) {
        progressBar.stop();
    }
    return testCaseVars;
}
async function synthesizeFromTestSuite(testSuite, options) {
    return synthesize({
        ...options,
        prompts: testSuite.prompts.map((prompt) => prompt.raw),
        tests: testSuite.tests || [],
    });
}
//# sourceMappingURL=testCases.js.map