"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.sanitizeProvider = sanitizeProvider;
// lodash imported for future use
const crypto_1 = require("crypto");
const drizzle_orm_1 = require("drizzle-orm");
const date_fns_1 = require("date-fns");
const database_1 = require("../database");
const tables_1 = require("../database/tables");
const envars_1 = require("../envars");
const utils_1 = require("../prompts/utils");
const clickhouseService_1 = require("../server/services/clickhouseService");
const evalResultsClickhouse_1 = require("../server/services/evalResultsClickhouse");
const types_1 = require("../types");
const providers_1 = require("../types/providers");
const json_1 = require("../util/json");
// Removes circular references from the provider object and ensures consistent format
function sanitizeProvider(provider) {
    try {
        if ((0, providers_1.isApiProvider)(provider)) {
            return {
                id: provider.id(),
                label: provider.label,
                ...(provider.config && {
                    config: JSON.parse((0, json_1.safeJsonStringify)(provider.config)),
                }),
            };
        }
        if ((0, providers_1.isProviderOptions)(provider)) {
            return {
                id: provider.id,
                label: provider.label,
                ...(provider.config && {
                    config: JSON.parse((0, json_1.safeJsonStringify)(provider.config)),
                }),
            };
        }
        if (typeof provider === 'object' && provider) {
            const providerObj = provider;
            return {
                id: typeof providerObj.id === 'function' ? providerObj.id() : providerObj.id,
                label: providerObj.label,
                ...(providerObj.config && {
                    config: JSON.parse((0, json_1.safeJsonStringify)(providerObj.config)),
                }),
            };
        }
    }
    catch { }
    return JSON.parse((0, json_1.safeJsonStringify)(provider));
}
class EvalResult {
    static async createFromEvaluateResult(evalId, result, opts) {
        const persist = opts?.persist == null ? true : opts.persist;
        const { prompt, error, score, latencyMs, success, provider, gradingResult, namedScores, cost, metadata, failureReason, testCase, isRedteam: isRedteamRaw, traceId, applicationId, projectId, workspaceId, sessionId, traceIds, } = result;
        const isRedteam = Number(isRedteamRaw) === 1 ? 1 : 0;
        const args = {
            id: (0, crypto_1.randomUUID)(),
            evalId,
            testCase: {
                ...testCase,
                ...(testCase?.provider && {
                    provider: sanitizeProvider(testCase.provider),
                }),
            },
            promptIdx: result.promptIdx,
            testIdx: result.testIdx,
            prompt,
            promptId: (0, utils_1.hashPrompt)(prompt),
            error: error?.toString() ?? null,
            success: success ? 1 : 0,
            score: score ?? 0,
            response: result.response || null,
            gradingResult: gradingResult ?? null,
            namedScores,
            provider: sanitizeProvider(provider),
            latencyMs,
            cost: cost ?? 0,
            metadata: metadata ?? {},
            failureReason,
            isRedteam,
            traceId,
            // Ensure applicationId is null if empty string or invalid
            applicationId: applicationId && applicationId.trim() !== '' ? applicationId : null,
            projectId,
            sessionId,
            workspaceId,
            traceIds,
        };
        if (persist) {
            const db = (0, database_1.getDb)();
            try {
                let projectId = args.projectId ?? null;
                let workspaceId = args.workspaceId ?? null;
                let evalRecord = null;
                let testRunId = null;
                if (!projectId) {
                    const [evalRecord_] = await db
                        .select({
                        projectId: tables_1.testRunsTable.projectId,
                        libraryId: tables_1.testRunsTable.libraryId,
                        workspaceId: tables_1.testRunsTable.workspaceId,
                        testRunId: tables_1.testRunsTable.id,
                        evalId: tables_1.testRunsToEvalsTable.evalId,
                        applicationIds: tables_1.testRunsTable.applicationIds,
                    })
                        .from(tables_1.testRunsTable)
                        .leftJoin(tables_1.testRunsToEvalsTable, (0, drizzle_orm_1.eq)(tables_1.testRunsTable.id, tables_1.testRunsToEvalsTable.testRunId))
                        .where((0, drizzle_orm_1.eq)(tables_1.testRunsToEvalsTable.evalId, evalId))
                        .limit(1);
                    projectId = evalRecord_?.projectId ?? null;
                    workspaceId = evalRecord_?.workspaceId ?? null;
                    evalRecord = evalRecord_;
                    args.projectId = projectId;
                    args.workspaceId = workspaceId;
                    testRunId = evalRecord_?.testRunId ?? null;
                    if (args.applicationId === null && Array.isArray(evalRecord_?.applicationIds) && evalRecord_?.applicationIds.length === 1) {
                        args.applicationId = evalRecord_?.applicationIds[0];
                    }
                }
                const [dbResult] = await db.insert(tables_1.evalResultsTable).values(args).returning();
                const hasTraceScope = (typeof args.traceId === 'string' && args.traceId.trim() !== '') ||
                    (Array.isArray(args.traceIds) && args.traceIds.length > 0);
                // Trace/session evals: ClickHouse + denorm batch insert in PulseEvaluationService (needs camelCase traceId).
                if (!hasTraceScope) {
                    const createdAt = dbResult.createdAt ?? new Date();
                    const ts = (0, date_fns_1.format)(createdAt, 'yyyy-MM-dd HH:mm:ss');
                    try {
                        await clickhouseService_1.clickHouseService.insert('eval_results', [
                            {
                                id: args.id,
                                created_at: ts,
                                updated_at: ts,
                                eval_id: args.evalId,
                                prompt_idx: args.promptIdx,
                                test_idx: args.testIdx,
                                test_case: args.testCase,
                                prompt: args.prompt,
                                prompt_id: args.promptId,
                                application_id: args.applicationId,
                                project_id: args.projectId,
                                workspace_id: args.workspaceId,
                                session_id: args.sessionId,
                                trace_id: args.traceId,
                                latency_ms: args.latencyMs,
                                cost: args.cost,
                                provider: args.provider,
                                response: args.response,
                                error: args.error,
                                failure_reason: args.failureReason,
                                success: args.success,
                                score: args.score,
                                grading_result: args.gradingResult,
                                named_scores: args.namedScores,
                                metadata: args.metadata,
                                is_redteam: args.isRedteam,
                                trace_ids: args.traceIds,
                                test_run_id: testRunId,
                            },
                        ]);
                    }
                    catch (clickhouseError) {
                        console.warn('Failed to insert eval result into ClickHouse eval_results:', clickhouseError);
                    }
                }
                // Trace/session: denorm batch in PulseEvaluationService after legacy CH insert.
                if (!hasTraceScope && (0, evalResultsClickhouse_1.isEvalDenormWritesEnabled)()) {
                    try {
                        await (0, evalResultsClickhouse_1.insertEvalDenormArtifacts)((0, evalResultsClickhouse_1.evalResultRecordToDenormInput)({
                            ...args,
                            id: dbResult.id,
                            createdAt: dbResult.createdAt,
                        }, { evalId: args.evalId, testRunId }), { skipPostgres: true });
                    }
                    catch (denormError) {
                        console.warn('Failed to insert eval denorm artifacts:', denormError);
                    }
                }
                const { failureReason: dbFailureReason, ...rest } = dbResult;
                return new EvalResult({
                    ...rest,
                    success: success ? 1 : 0,
                    persisted: true,
                    projectId: projectId ?? null,
                    workspaceId: workspaceId ?? null,
                    applicationId: args.applicationId ?? null,
                    traceId: args.traceId ?? null,
                    isRedteam: args.isRedteam ?? 0,
                    sessionId: args.sessionId ?? null,
                    failureReason: (dbFailureReason ?? types_1.ResultFailureReason.NONE),
                });
            }
            catch (error) {
                console.error(error);
                throw new Error(error?.message || 'Error creating eval result');
            }
        }
        return new EvalResult(args);
    }
    static async createManyFromEvaluateResult(results, evalId) {
        const db = (0, database_1.getDb)();
        const returnResults = [];
        await db.transaction(async (tx) => {
            for (const result of results) {
                const dbResult = await tx
                    .insert(tables_1.evalResultsTable)
                    .values({ ...result, success: result.success ? 1 : 0, evalId, id: (0, crypto_1.randomUUID)() })
                    .returning();
                const { failureReason: dbFailureReason, ...rest } = dbResult[0];
                returnResults.push(new EvalResult({ ...rest, persisted: true, failureReason: (dbFailureReason ?? types_1.ResultFailureReason.NONE) }));
            }
        });
        return returnResults;
    }
    static async findById(id) {
        const db = (0, database_1.getDb)();
        const result = await db.select().from(tables_1.evalResultsTable).where((0, drizzle_orm_1.eq)(tables_1.evalResultsTable.id, id));
        if (result.length > 0) {
            const { failureReason: dbFailureReason, ...rest } = result[0];
            return new EvalResult({ ...rest, persisted: true, failureReason: (dbFailureReason ?? types_1.ResultFailureReason.NONE) });
        }
        return null;
    }
    static async findManyByEvalId(evalId, opts) {
        const db = (0, database_1.getDb)();
        const whereClause = (0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(tables_1.evalResultsTable.evalId, evalId), opts?.testIdx == null ? undefined : (0, drizzle_orm_1.eq)(tables_1.evalResultsTable.testIdx, opts.testIdx));
        if (opts?.excludePromptResponse) {
            const results = await db
                .select({
                id: tables_1.evalResultsTable.id,
                createdAt: tables_1.evalResultsTable.createdAt,
                applicationId: tables_1.evalResultsTable.applicationId,
                sessionId: tables_1.evalResultsTable.sessionId,
                traceId: tables_1.evalResultsTable.traceId,
                projectId: tables_1.evalResultsTable.projectId,
                provider: tables_1.evalResultsTable.provider,
                latencyMs: tables_1.evalResultsTable.latencyMs,
                cost: tables_1.evalResultsTable.cost,
                error: tables_1.evalResultsTable.error,
                failureReason: tables_1.evalResultsTable.failureReason,
                success: tables_1.evalResultsTable.success,
                score: tables_1.evalResultsTable.score,
                gradingResult: tables_1.evalResultsTable.gradingResult,
                namedScores: tables_1.evalResultsTable.namedScores,
                metadata: tables_1.evalResultsTable.metadata,
                isRedteam: tables_1.evalResultsTable.isRedteam,
                workspaceId: tables_1.evalResultsTable.workspaceId,
                traceIds: tables_1.evalResultsTable.traceIds,
            })
                .from(tables_1.evalResultsTable)
                .where(whereClause);
            return results.map((result) => {
                const { failureReason: dbFailureReason, ...rest } = result;
                return new EvalResult({
                    ...rest,
                    prompt: { raw: '', label: '' },
                    response: null,
                    persisted: true,
                    failureReason: (dbFailureReason ?? types_1.ResultFailureReason.NONE),
                });
            });
        }
        const results = await db.select().from(tables_1.evalResultsTable).where(whereClause);
        return results.map((result) => {
            const { failureReason: dbFailureReason, ...rest } = result;
            return new EvalResult({
                ...rest,
                persisted: true,
                failureReason: (dbFailureReason ?? types_1.ResultFailureReason.NONE),
            });
        });
    }
    static async findManyByEvalIdAndTestIndices(evalId, testIndices) {
        if (!testIndices.length) {
            return [];
        }
        const db = (0, database_1.getDb)();
        const results = await db
            .select()
            .from(tables_1.evalResultsTable)
            .where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(tables_1.evalResultsTable.evalId, evalId), testIndices.length === 1
            ? (0, drizzle_orm_1.eq)(tables_1.evalResultsTable.testIdx, testIndices[0])
            : (0, drizzle_orm_1.inArray)(tables_1.evalResultsTable.testIdx, testIndices)));
        return results.map((result) => {
            const { failureReason: dbFailureReason, ...rest } = result;
            return new EvalResult({ ...rest, persisted: true, failureReason: (dbFailureReason ?? types_1.ResultFailureReason.NONE) });
        });
    }
    // This is a generator that yields batches of results from the database
    // These are batched by test Id, not just results to ensure we get all results for a given test
    static async *findManyByEvalIdBatched(evalId, opts) {
        const db = (0, database_1.getDb)();
        const batchSize = opts?.batchSize || 100;
        let offset = 0;
        while (true) {
            const results = await db
                .select()
                .from(tables_1.evalResultsTable)
                .where((0, drizzle_orm_1.and)((0, drizzle_orm_1.eq)(tables_1.evalResultsTable.evalId, evalId), (0, drizzle_orm_1.gte)(tables_1.evalResultsTable.testIdx, offset), (0, drizzle_orm_1.lt)(tables_1.evalResultsTable.testIdx, offset + batchSize)))
                .execute();
            if (results.length === 0) {
                break;
            }
            yield results.map((result) => new EvalResult({ ...result, persisted: true, failureReason: (result.failureReason ?? types_1.ResultFailureReason.NONE) }));
            offset += batchSize;
        }
    }
    constructor(opts) {
        this.id = opts.id;
        this.evalId = opts.evalId;
        this.promptIdx = opts.promptIdx;
        this.testIdx = opts.testIdx;
        this.testCase = opts.testCase;
        this.prompt = opts.prompt;
        this.promptId = opts.promptId || (0, utils_1.hashPrompt)(opts.prompt) || '';
        this.error = opts.error;
        this.score = opts.score;
        this.success = opts.success;
        this.response = opts.response || undefined;
        this.gradingResult = opts.gradingResult;
        this.namedScores = opts.namedScores || {};
        this.provider = opts.provider;
        this.latencyMs = opts.latencyMs || 0;
        this.cost = opts.cost || 0;
        this.metadata = opts.metadata || {};
        this.failureReason = opts.failureReason;
        this.persisted = opts.persisted || false;
        this.config = opts.config || null;
        this.createdAt = opts.createdAt;
        this.projectId = opts.projectId;
        this.applicationId = opts.applicationId;
        this.workspaceId = opts.workspaceId;
        this.traceId = opts.traceId;
        this.isRedteam = opts.isRedteam;
        this.sessionId = opts.sessionId;
        this.traceIds = opts.traceIds;
    }
    async save() {
        const db = (0, database_1.getDb)();
        const resultData = {
            ...this,
            success: this.success ? 1 : 0,
            updatedAt: (0, drizzle_orm_1.sql) `CURRENT_TIMESTAMP`,
        };
        // If already persisted, update the record, otherwise insert a new one
        if (this.persisted) {
            await db
                .update(tables_1.evalResultsTable)
                .set(resultData) // Use the resultData with explicit success conversion
                .where((0, drizzle_orm_1.eq)(tables_1.evalResultsTable.id, this.id));
        }
        else {
            // Insert new record with resultData and return the inserted row
            const result = await db.insert(tables_1.evalResultsTable).values(resultData).returning();
            this.id = result[0].id;
            this.persisted = true;
        }
    }
    toEvaluateResult(opts) {
        const shouldStripPromptText = (0, envars_1.getEnvBool)('TRUSYS_STRIP_PROMPT_TEXT', false);
        const shouldStripResponseOutput = (0, envars_1.getEnvBool)('TRUSYS_STRIP_RESPONSE_OUTPUT', false);
        const shouldStripTestVars = (0, envars_1.getEnvBool)('TRUSYS_STRIP_TEST_VARS', false);
        const shouldStripGradingResult = (0, envars_1.getEnvBool)('TRUSYS_STRIP_GRADING_RESULT', false);
        const shouldStripMetadata = (0, envars_1.getEnvBool)('TRUSYS_STRIP_METADATA', false);
        const response = shouldStripResponseOutput && this.response
            ? {
                ...this.response,
                output: '[output stripped]',
            }
            : this.response;
        const prompt = shouldStripPromptText
            ? {
                ...this.prompt,
                raw: '[prompt stripped]',
            }
            : this.prompt;
        const testCase = shouldStripTestVars
            ? {
                ...this.testCase,
                vars: undefined,
            }
            : this.testCase;
        const out = {
            cost: this.cost,
            description: this.description || undefined,
            evalId: this.evalId,
            error: this.error || undefined,
            gradingResult: shouldStripGradingResult ? null : this.gradingResult,
            id: this.id,
            latencyMs: this.latencyMs,
            namedScores: this.namedScores,
            prompt,
            promptId: this.promptId,
            promptIdx: this.promptIdx,
            provider: { id: this.provider.id, label: this.provider.label },
            response,
            score: this.score,
            success: this.success ? 1 : 0,
            testCase,
            testIdx: this.testIdx,
            vars: shouldStripTestVars ? {} : this?.testCase?.vars || {},
            metadata: shouldStripMetadata ? {} : this.metadata,
            failureReason: this.failureReason,
        };
        if (opts?.excludePromptResponse) {
            delete out.prompt;
            delete out.response;
        }
        return out;
    }
}
exports.default = EvalResult;
//# sourceMappingURL=evalResult.js.map