"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.datasetService = exports.DatasetService = exports.DatasetRestrictionError = void 0;
const decimal_js_1 = __importDefault(require("decimal.js"));
const dedent_1 = __importDefault(require("dedent"));
const drizzle_orm_1 = require("drizzle-orm");
const node_fs_1 = __importDefault(require("node:fs"));
const node_os_1 = __importDefault(require("node:os"));
const node_path_1 = __importDefault(require("node:path"));
const uuid_1 = require("uuid");
const constants_1 = require("../../constants");
const tables_1 = require("../../database/tables");
const database_1 = require("../../database");
const tables_2 = require("../../database/tables");
const logger_1 = __importDefault(require("../../logger"));
const synthesis_1 = require("../../testCase/synthesis");
const billing_logger_1 = __importDefault(require("./billing/billing-logger"));
const get_upgrade_meta_1 = require("../../util/get-upgrade-meta");
const restriction_message_1 = require("../constants/restriction.message");
const callControlPlaneApi_1 = require("../../util/callContolPlane/callControlPlaneApi");
const convertTextToAudioWithProvider_1 = require("../../util/convertTextToAudioWithProvider/convertTextToAudioWithProvider");
const uploadFilesToAzure_1 = require("../../util/uploadFilesToAzure");
const generateDocumentDatasetRowsForEvaluation_1 = require("./datasetDocumentSynthesis/generateDocumentDatasetRowsForEvaluation");
const datasetDynamicDatasetLanguage_1 = require("./datasetDynamicDatasetLanguage");
const datasetEncryptGeneratedRows_1 = require("./datasetEncryptGeneratedRows");
const lodash_1 = __importDefault(require("lodash"));
class DatasetRestrictionError extends Error {
    constructor(message, restrictionType, showToast, toastMetadata) {
        super(message);
        this.restrictionType = restrictionType;
        this.showToast = showToast;
        this.toastMetadata = toastMetadata;
        this.name = 'DatasetRestrictionError';
    }
}
exports.DatasetRestrictionError = DatasetRestrictionError;
function hasExplicitFieldsToGenerate(fieldsToGenerate) {
    return Array.isArray(fieldsToGenerate) && fieldsToGenerate.length > 0;
}
function filterDatasetFieldsByNames(allFields, fieldsToGenerate) {
    const fields = Array.isArray(allFields) ? allFields : [];
    if (!hasExplicitFieldsToGenerate(fieldsToGenerate)) {
        return fields;
    }
    return fields.filter((f) => fieldsToGenerate.includes(String(f?.name)));
}
function resolveFieldNamesToGenerate(fieldsToGenerate) {
    if (!hasExplicitFieldsToGenerate(fieldsToGenerate)) {
        return undefined;
    }
    return fieldsToGenerate.map((f) => String(f));
}
function filterTestsVarsByFieldNames(tests, fieldsToGenerate) {
    if (!hasExplicitFieldsToGenerate(fieldsToGenerate)) {
        return tests ?? [];
    }
    return (tests ?? []).map((t) => {
        const vars = t?.vars && typeof t.vars === 'object' ? t.vars : {};
        const nextVars = Object.fromEntries(Object.entries(vars).filter(([k]) => fieldsToGenerate.includes(String(k))));
        return {
            ...t,
            vars: nextVars,
        };
    });
}
function filterGeneratedRowsByFieldNames(generatedData, fieldsToGenerate) {
    if (!hasExplicitFieldsToGenerate(fieldsToGenerate) || !Array.isArray(generatedData)) {
        return generatedData;
    }
    const allow = new Set(fieldsToGenerate.map((f) => String(f)));
    return generatedData.map((row) => {
        if (!row || typeof row !== 'object' || Array.isArray(row)) {
            return row;
        }
        const next = {};
        for (const k of Object.keys(row)) {
            if (allow.has(k)) {
                next[k] = row[k];
            }
        }
        return next;
    });
}
function resolveDocumentIncludeExpectedOutput(datasetConfig, fieldNamesToGenerate) {
    if (datasetConfig?.includeExpectedOutput === false) {
        return false;
    }
    if (!fieldNamesToGenerate || fieldNamesToGenerate.length === 0) {
        return true;
    }
    return fieldNamesToGenerate.some((name) => name === 'expected_output' || name === 'expectedOutput');
}
class DatasetService {
    async applyVoiceFieldsToGeneratedDatasetRows(params) {
        const { datasetId, orgId, projectId, datasetConfig, datasetFields, generatedData, token } = params;
        const voiceFields = Array.isArray(datasetFields)
            ? datasetFields.filter((f) => f?.type === constants_1.DATASET_VAR_TYPE.VOICE && f?.name)
            : [];
        if (voiceFields.length === 0 || !Array.isArray(generatedData) || generatedData.length === 0) {
            return;
        }
        const audioTmpDir = node_fs_1.default.mkdtempSync(node_path_1.default.join(node_os_1.default.tmpdir(), 'trusys-dataset-audio-'));
        const folderName = `workspace/${orgId}/project/${projectId}/dataset/${datasetId}`;
        const defaultLanguage = (0, datasetDynamicDatasetLanguage_1.getDatasetDefaultTtsLocale)(datasetConfig);
        const filesToUpload = [];
        const fileNameByRowAndField = new Map();
        const CONCURRENCY_LIMIT = 5;
        for (let start = 0; start < generatedData.length; start += CONCURRENCY_LIMIT) {
            const slice = generatedData.slice(start, start + CONCURRENCY_LIMIT);
            await Promise.all(slice.map(async (row, idx) => {
                const rowIndex = start + idx + 1;
                if (!row || typeof row !== 'object') {
                    return;
                }
                await Promise.all(voiceFields.map(async (field) => {
                    const fieldName = String(field.name);
                    const rawValue = row[fieldName];
                    if (rawValue == null) {
                        return;
                    }
                    // If already shaped as a voice object, leave it.
                    if (typeof rawValue === 'object' && rawValue?.type === 'voice') {
                        return;
                    }
                    const value = typeof rawValue === 'string' ? rawValue : String(rawValue);
                    const outputFormat = String(field?.outputFormat || 'wav').toLowerCase().trim() || 'wav';
                    const sampleRate = String(field?.sampleRate || '44100').trim() || '44100';
                    const fieldLanguage = field?.language?.SpeechSynthesisLanguage || defaultLanguage || 'en-US';
                    try {
                        const wavFilePath = await (0, convertTextToAudioWithProvider_1.convertTextToAudioFile)(value, orgId, outputFormat, sampleRate, fieldName, rowIndex, audioTmpDir, fieldLanguage);
                        const fileName = node_path_1.default.basename(wavFilePath);
                        filesToUpload.push({ wavFilePath, fileName });
                        fileNameByRowAndField.set(`${rowIndex}:${fieldName}`, fileName);
                    }
                    catch (error) {
                        logger_1.default.error(`Error generating voice for dataset=${datasetId} field=${fieldName}: ${error instanceof Error ? error.message : String(error)}`);
                    }
                }));
            }));
        }
        if (filesToUpload.length > 0) {
            const batchSize = 10;
            const batches = [];
            for (let i = 0; i < filesToUpload.length; i += batchSize) {
                batches.push((0, uploadFilesToAzure_1.uploadAudioFiles)(filesToUpload.slice(i, i + batchSize), token, folderName));
            }
            const urlMaps = await Promise.all(batches);
            const urlMap = urlMaps.reduce((acc, curr) => ({ ...acc, ...curr }), {});
            for (let i = 0; i < generatedData.length; i++) {
                const rowIndex = i + 1;
                const row = generatedData[i];
                if (!row || typeof row !== 'object') {
                    continue;
                }
                for (const field of voiceFields) {
                    const fieldName = String(field.name);
                    const rawValue = row[fieldName];
                    if (rawValue == null) {
                        continue;
                    }
                    if (typeof rawValue === 'object' && rawValue?.type === 'voice') {
                        continue;
                    }
                    const value = typeof rawValue === 'string' ? rawValue : String(rawValue);
                    const fileName = fileNameByRowAndField.get(`${rowIndex}:${fieldName}`);
                    row[fieldName] = {
                        type: 'voice',
                        value,
                        voiceUrl: fileName ? urlMap[fileName] || null : null,
                    };
                }
            }
            return;
        }
        // No files generated; still wrap voice fields for consistency.
        for (let i = 0; i < generatedData.length; i++) {
            const row = generatedData[i];
            if (!row || typeof row !== 'object') {
                continue;
            }
            for (const field of voiceFields) {
                const fieldName = String(field.name);
                const rawValue = row[fieldName];
                if (rawValue == null) {
                    continue;
                }
                if (typeof rawValue === 'object' && rawValue?.type === 'voice') {
                    continue;
                }
                const value = typeof rawValue === 'string' ? rawValue : String(rawValue);
                row[fieldName] = { type: 'voice', value, voiceUrl: null };
            }
        }
    }
    /**
     * Validate workspace restrictions for dataset generation
     * Throws DatasetRestrictionError if restrictions are not met
     */
    async validateDatasetRestrictions(orgId) {
        const { restrictionsData, isEnterprise, wallet, notifications, thresholds } = await (0, get_upgrade_meta_1.getUpgradeMetaByOrgId)(orgId, {
            fetchMetricUsage: false,
        });
        // Check dataset limit restriction
        if (!isEnterprise && restrictionsData?.datasets > 0) {
            throw new DatasetRestrictionError(restriction_message_1.ERROR_RESTRICTION_MESSAGE.upgrade.title, 'upgrade', true, {
                text: restriction_message_1.ERROR_RESTRICTION_MESSAGE.upgrade.message,
                type: 'error',
            });
        }
        // Check wallet credits restriction
        if (!isEnterprise && new decimal_js_1.default(wallet?.credits ?? 0).lte(0)) {
            if (notifications?.canSendWalletUsageExceeded) {
                try {
                    await (0, callControlPlaneApi_1.callControlPlane)(`/trusys-emails/limit-reached`, {
                        method: 'POST',
                        params: {
                            orgId,
                            type: 'Wallet',
                            currentUsage: wallet?.credits ?? 0,
                        },
                    }, '');
                }
                catch (error) {
                    logger_1.default.error(`Error sending wallet limit reached email: ${error}`);
                }
            }
            throw new DatasetRestrictionError(restriction_message_1.ERROR_RESTRICTION_MESSAGE.lowCredits.title, 'lowCredits', true, {
                text: restriction_message_1.ERROR_RESTRICTION_MESSAGE.lowCredits.message,
                type: 'error',
            });
        }
        // Send low wallet balance notification (non-blocking)
        if (!isEnterprise &&
            new decimal_js_1.default(wallet?.credits ?? 0).lte(thresholds?.walletNotificationThreshold ?? 0) &&
            notifications?.canSendLowWalletBalance) {
            try {
                await (0, callControlPlaneApi_1.callControlPlane)(`/trusys-emails/limit-reaching`, {
                    method: 'POST',
                    params: {
                        orgId,
                        type: 'Wallet',
                        currentUsage: wallet?.credits ?? 0,
                    },
                }, '');
            }
            catch (error) {
                logger_1.default.error(`Error sending low wallet balance email: ${error}`);
            }
        }
    }
    /**
     * Generate dataset from prompts/tests using synthesis
     */
    async generateDatasetFromPrompts(params, options) {
        const { promptLibraryId, tests, provider, instructions, numSamples, prompt = '{{prompt}}', fields, language: languageParam = 'English', orgId, datasetId, } = params;
        const language = typeof languageParam === 'string' && languageParam.trim()
            ? languageParam.trim()
            : (0, datasetDynamicDatasetLanguage_1.getDatasetLanguageLabelForSynthesis)({ language: languageParam });
        // Validate restrictions unless explicitly skipped
        if (!options?.skipRestrictions) {
            await this.validateDatasetRestrictions(orgId);
        }
        const db = await (0, database_1.getDb)();
        // Fetch prompts from the prompt library if promptLibraryId is provided
        let prompts = [];
        if (promptLibraryId) {
            prompts = await db
                .select({
                prompt: tables_1.promptsTable.prompt,
            })
                .from(tables_1.promptsLibraryPromptsTable)
                .innerJoin(tables_1.promptsTable, (0, drizzle_orm_1.eq)(tables_1.promptsLibraryPromptsTable.promptId, tables_1.promptsTable.id))
                .where((0, drizzle_orm_1.eq)(tables_1.promptsLibraryPromptsTable.libraryId, promptLibraryId));
        }
        if (lodash_1.default.isString(prompt) && !promptLibraryId) {
            prompts = [{ prompt: '{{prompt}}' }];
        }
        if (!Array.isArray(prompts) || prompts.length === 0) {
            throw new Error(promptLibraryId
                ? 'The selected prompt library has no prompts. Add at least one prompt before generating a dataset.'
                : 'Dataset generation requires at least one prompt or a prompt library.');
        }
        const testSuite = {
            prompts: prompts?.map((p) => ({ raw: p?.prompt, label: p?.prompt })),
            tests: tests || [],
            providers: [],
        };
        const includefillers = fields
            ?.filter((item) => item?.type === constants_1.DATASET_VAR_TYPE.VOICE && item?.voiceOptions?.includeFillerWords)
            .map((item) => item.name) ?? [];
        let updatedInstructions = instructions;
        if (includefillers.length > 0 || language) {
            const numbered = [];
            if (instructions) {
                numbered.push(instructions);
            }
            if (includefillers.length > 0) {
                numbered.push(`Make the generated values sound like human speech. Only include filler words and hesitation markers in the following variables: ${includefillers.join(', ')}.`);
            }
            if (language) {
                numbered.push(`Create the variables in this language: '${language}'.`);
            }
            numbered.push((0, dedent_1.default) `
          Output hygiene (applies to every variable value):
          - Return clean values only (no field names, no labels like "phone:", no extra commentary).
          - Do not add trailing or leading punctuation or wrappers (avoid extra ",", ";", ":", "'", '"', parentheses).
          - Do not repeat separators/delimiters or emit multiple alternative formats for the same value.
          - Preserve meaning while keeping formatting natural (e.g. numbers, emails, URLs should look realistic and valid).`);
            updatedInstructions = (0, dedent_1.default) `
        Generate the variables using the following instructions:
        ${numbered.map((line, i) => `${i + 1}. ${line}`).join('\n')}
      `;
        }
        if (Array.isArray(params.previousDatasetGenerated) && params.previousDatasetGenerated.length > 0) {
            const previousDatasetsBlock = params.previousDatasetGenerated
                .slice(0, 20)
                .map((row) => {
                try {
                    return JSON.stringify(row, null, 2);
                }
                catch {
                    return String(row);
                }
            })
                .join('\n\n');
            updatedInstructions = `${updatedInstructions ?? ''}\n\n${(0, dedent_1.default) `
        Your goal is to generate a batch of datasets where:
        - some datasets may be similar in content to previous ones
        - some datasets should be clearly different from previous datasets

        It is acceptable if a few datasets overlap with previous ones, but the batch should not feel repetitive overall.

        When creating different datasets, vary one or more of (Be smart enough to vary these based on the dataset required):
        - user persona
        - tone
        - ambiguity
        - level of detail
        - edge-case behavior

        these are prevous generated datasets: ${previousDatasetsBlock}
      `}`;
        }
        const datasetTokenUsage = {
            cost: new decimal_js_1.default(0),
            tokenUsage: {
                prompt: new decimal_js_1.default(0),
                completion: new decimal_js_1.default(0),
                total: new decimal_js_1.default(0),
                cached: new decimal_js_1.default(0),
            },
        };
        const results = await (0, synthesis_1.synthesizeFromTestSuite)(testSuite, {
            instructions: updatedInstructions,
            numPersonas: 5,
            numTestCasesPerPersona: Number.parseInt(String(numSamples)) === 1
                ? Number.parseInt(String(numSamples))
                : Math.floor(Number.parseInt(String(numSamples)) || 15) / 5,
            provider,
            apiCall: true,
            workspaceId: orgId,
        }, datasetTokenUsage);
        // synthesizeFromTestSuite with apiCall: true returns { personas: string[], testCaseVars: VarMapping[] }
        // but can also return VarMapping[] if apiCall is false, so we handle both cases
        const testCaseVars = Array.isArray(results)
            ? results
            : results.testCaseVars || [];
        const personas = Array.isArray(results)
            ? []
            : results.personas || [];
        // Log billing if token usage occurred
        if (datasetTokenUsage?.tokenUsage?.total.gt(0) ||
            datasetTokenUsage?.cost.gt(0) ||
            datasetTokenUsage?.tokenUsage?.prompt.gt(0) ||
            datasetTokenUsage?.tokenUsage?.completion.gt(0) ||
            datasetTokenUsage?.tokenUsage?.cached.gt(0)) {
            billing_logger_1.default.logDatasetUsage({
                cost: datasetTokenUsage.cost.toNumber(),
                tokenUsage: {
                    prompt: datasetTokenUsage.tokenUsage.prompt.toNumber(),
                    completion: datasetTokenUsage.tokenUsage.completion.toNumber(),
                    total: datasetTokenUsage.tokenUsage.total.toNumber(),
                    cached: datasetTokenUsage.tokenUsage.cached.toNumber(),
                },
                workspaceId: orgId,
                datasetId: datasetId || '',
                metadata: {
                    promptLibraryId,
                },
                provider: 'trusys',
                model: 'gpt-4.1',
            });
        }
        return {
            testCaseVars,
            totalRows: testCaseVars.length,
            personas,
            tokenUsages: datasetTokenUsage,
        };
    }
    /**
     * Save dataset records to the database
     */
    async saveDatasetRecords(params) {
        const { datasetId, data, append = false } = params;
        const db = await (0, database_1.getDb)();
        if (!Array.isArray(data) || data.length === 0) {
            return [];
        }
        // Delete existing records only if not appending
        if (!append) {
            await db.delete(tables_2.datasetRecordsTable).where((0, drizzle_orm_1.eq)(tables_2.datasetRecordsTable.datasetId, datasetId));
        }
        // If appending, get the current max recordIndex to continue from
        /*     let startIndex = 0;
            if (append) {
              const maxRecord = await db
                .select({ recordIndex: datasetRecordsTable.recordIndex })
                .from(datasetRecordsTable)
                .where(eq(datasetRecordsTable.datasetId, datasetId))
                .orderBy(desc(datasetRecordsTable.recordIndex))
                .limit(1);
              startIndex = (maxRecord[0]?.recordIndex ?? -1) + 1;
            }
        */
        // Insert new records
        const recordInserts = data.map((record, index) => ({
            id: (0, uuid_1.v4)(),
            datasetId,
            recordData: record,
            recordIndex: index,
        }));
        await db.insert(tables_2.datasetRecordsTable).values(recordInserts);
        return recordInserts.map((r) => r.id);
    }
    /**
     * Map test run to dataset records
     */
    async mapTestRunToDatasetRecords(params) {
        const { testRunId, datasetId, datasetRecordIds } = params;
        const db = await (0, database_1.getDb)();
        if (!datasetRecordIds || datasetRecordIds.length === 0) {
            logger_1.default.warn(`No dataset record IDs provided for test run ${testRunId}`);
            return;
        }
        // Insert mappings
        const mappingInserts = datasetRecordIds.map((recordId) => ({
            testRunId,
            datasetRecordId: recordId,
            datasetId,
        }));
        await db
            .insert(tables_2.testRunsToDatasetRecordsTable)
            .values(mappingInserts)
            .onConflictDoNothing();
        logger_1.default.info(`Mapped ${mappingInserts.length} dataset records to test run ${testRunId}`);
    }
    /**
     * Generate dataset for a dataset with generateOnEvaluation = true
     * This is used when a test run needs to generate a dataset on-the-fly
     * Throws DatasetRestrictionError if restrictions are not met
     */
    async generateDatasetForEvaluation(params) {
        const { datasetId, datasetConfig, orgId, projectId: _projectId } = params;
        // Validate restrictions before generating
        await this.validateDatasetRestrictions(orgId);
        const db = await (0, database_1.getDb)();
        // Fetch dataset details
        const [dataset] = await db
            .select()
            .from(tables_2.trusysDatasetsTable)
            .where((0, drizzle_orm_1.eq)(tables_2.trusysDatasetsTable.id, datasetId));
        if (!dataset) {
            throw new Error(`Dataset with id ${datasetId} not found`);
        }
        if (Array.isArray(params.fieldsToGenerate) && params.fieldsToGenerate.length === 0) {
            return {
                datasetRecordIds: [],
                data: [],
            };
        }
        const allFields = (Array.isArray(dataset.fields) ? dataset.fields : []);
        const filteredFields = filterDatasetFieldsByNames(allFields, params.fieldsToGenerate);
        const fieldNamesToGenerate = resolveFieldNamesToGenerate(params.fieldsToGenerate);
        let generatedData = [];
        // Check dataset type and generate accordingly
        if (dataset.type === constants_1.DATASET_TYPE.GENERATE) {
            const previousDatasetGenerated = await db
                .select({ recordData: tables_2.datasetRecordsTable.recordData })
                .from(tables_2.datasetRecordsTable)
                .where((0, drizzle_orm_1.eq)(tables_2.datasetRecordsTable.datasetId, datasetId))
                .orderBy((0, drizzle_orm_1.desc)(tables_2.datasetRecordsTable.recordIndex))
                .limit(20);
            // Extract prompts from test suite if available, otherwise use promptLibraryId
            let promptText = '';
            if (params.prompts && params.prompts.length > 0) {
                // Handle different prompt formats from test suite
                const firstPrompt = params.prompts[0];
                if (typeof firstPrompt === 'string') {
                    promptText = firstPrompt;
                }
                else if (Array.isArray(firstPrompt)) {
                    // For conversation builders, extract text from first turn
                    promptText = firstPrompt[0]?.content || firstPrompt[0] || '';
                }
                else if (firstPrompt?.raw) {
                    promptText = firstPrompt.raw;
                }
                else if (firstPrompt?.label) {
                    promptText = firstPrompt.label;
                }
                else if (firstPrompt?.content) {
                    promptText = firstPrompt.content;
                }
            }
            // Generate from prompts/tests (skip restrictions since we already validated)
            const filteredTests = filterTestsVarsByFieldNames(datasetConfig?.tests || [], params.fieldsToGenerate);
            const generationResult = await this.generateDatasetFromPrompts({
                promptLibraryId: datasetConfig?.promptLibraryId,
                prompt: promptText, // Use prompt from test suite if available
                tests: filteredTests,
                provider: datasetConfig?.provider,
                instructions: datasetConfig?.instructions || '',
                numSamples: datasetConfig?.numSamples || 15,
                fields: filteredFields,
                language: (0, datasetDynamicDatasetLanguage_1.getDatasetLanguageLabelForSynthesis)(datasetConfig),
                orgId,
                workspaceId: orgId,
                datasetId,
                previousDatasetGenerated: previousDatasetGenerated.map((r) => r.recordData),
            }, { skipRestrictions: true });
            generatedData = generationResult.testCaseVars;
        }
        else if (dataset.type === constants_1.DATASET_TYPE.GENERATE_FROM_DOCUMENTS) {
            if (datasetConfig?.generateOnEvaluation !== true) {
                throw new Error('Document-based dataset generation on evaluation requires generateOnEvaluation to be enabled.');
            }
            const previousDocumentRows = await db
                .select({ recordData: tables_2.datasetRecordsTable.recordData })
                .from(tables_2.datasetRecordsTable)
                .where((0, drizzle_orm_1.eq)(tables_2.datasetRecordsTable.datasetId, datasetId))
                .orderBy((0, drizzle_orm_1.desc)(tables_2.datasetRecordsTable.recordIndex))
                .limit(20);
            const numSamples = Number(datasetConfig?.numSamples) || 15;
            const includeExpectedOutput = resolveDocumentIncludeExpectedOutput(datasetConfig, fieldNamesToGenerate);
            const { rows, tokenUsages } = await (0, generateDocumentDatasetRowsForEvaluation_1.generateDocumentDatasetRowsForEvaluation)({
                datasetConfig,
                numSamples,
                includeExpectedOutput,
                fieldsToGenerate: fieldNamesToGenerate,
                previousRecordData: previousDocumentRows.map((r) => r.recordData),
            });
            generatedData = rows;
            if (tokenUsages?.tokenUsage?.total.gt(0) ||
                tokenUsages?.cost.gt(0) ||
                tokenUsages?.tokenUsage?.prompt.gt(0) ||
                tokenUsages?.tokenUsage?.completion.gt(0) ||
                tokenUsages?.tokenUsage?.cached.gt(0)) {
                billing_logger_1.default.logDatasetUsage({
                    cost: tokenUsages.cost.toNumber(),
                    tokenUsage: {
                        prompt: tokenUsages.tokenUsage.prompt.toNumber(),
                        completion: tokenUsages.tokenUsage.completion.toNumber(),
                        total: tokenUsages.tokenUsage.total.toNumber(),
                        cached: tokenUsages.tokenUsage.cached.toNumber(),
                    },
                    workspaceId: orgId,
                    datasetId,
                    metadata: {
                        projectId: params.projectId,
                        datasetId,
                        generationMethod: 'documents_evaluation',
                    },
                    provider: 'trusys',
                    model: 'gpt-4.1',
                });
            }
        }
        generatedData = filterGeneratedRowsByFieldNames(generatedData, params.fieldsToGenerate);
        const capRows = params.fromConversationBuilder === true &&
            typeof params.maxGeneratedRows === 'number' &&
            Number.isFinite(params.maxGeneratedRows) &&
            params.maxGeneratedRows > 0
            ? Math.floor(params.maxGeneratedRows)
            : null;
        if (capRows != null && Array.isArray(generatedData) && generatedData.length > capRows) {
            generatedData = generatedData.slice(0, capRows);
        }
        const fieldsForEncryption = hasExplicitFieldsToGenerate(params.fieldsToGenerate)
            ? filteredFields
            : allFields;
        if (Array.isArray(generatedData) &&
            generatedData.length > 0 &&
            fieldsForEncryption.some((f) => f?.includeEncryptField)) {
            generatedData = await (0, datasetEncryptGeneratedRows_1.encryptGeneratedDatasetRows)(generatedData, fieldsForEncryption, (0, datasetDynamicDatasetLanguage_1.getDatasetLanguageForPresidio)(datasetConfig));
        }
        await this.applyVoiceFieldsToGeneratedDatasetRows({
            datasetId,
            orgId,
            projectId: params.projectId,
            datasetConfig,
            datasetFields: hasExplicitFieldsToGenerate(params.fieldsToGenerate) ? filteredFields : allFields,
            generatedData,
            token: params.token,
        });
        // Save the generated records (append=true to preserve records from other test runs)
        const datasetRecordIds = await this.saveDatasetRecords({
            datasetId,
            data: generatedData,
            orgId,
            projectId: params.projectId,
            append: true, // Append records instead of replacing, since multiple test runs can use the same dataset
        });
        return {
            datasetRecordIds,
            data: generatedData,
        };
    }
    /**
     * Get dataset records for a dataset
     */
    async getDatasetRecords(datasetId) {
        const db = await (0, database_1.getDb)();
        const records = await db
            .select({
            recordData: tables_2.datasetRecordsTable.recordData,
            recordId: tables_2.datasetRecordsTable.id,
        })
            .from(tables_2.datasetRecordsTable)
            .where((0, drizzle_orm_1.eq)(tables_2.datasetRecordsTable.datasetId, datasetId));
        return records.map((record) => {
            const recordData = record.recordData;
            return {
                ...recordData,
                datasetRecordId: record.recordId,
            };
        });
    }
}
exports.DatasetService = DatasetService;
// Export singleton instance
exports.datasetService = new DatasetService();
//# sourceMappingURL=datasetService.js.map