"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.buildBackfillOrderByClause = buildBackfillOrderByClause;
exports.buildBackfillLimitClause = buildBackfillLimitClause;
exports.buildBackfillLimitOffsetClause = buildBackfillLimitOffsetClause;
exports.rowToResumeCursor = rowToResumeCursor;
exports.buildBackfillCursorPredicate = buildBackfillCursorPredicate;
exports.resolveResumeCursor = resolveResumeCursor;
exports.resolveResumeOffset = resolveResumeOffset;
exports.computeOffsetShardRanges = computeOffsetShardRanges;
exports.backfillEvalResultsDenorm = backfillEvalResultsDenorm;
const node_fs_1 = require("node:fs");
const drizzle_orm_1 = require("drizzle-orm");
const database_1 = require("../../../database");
const logger_1 = __importDefault(require("../../../logger"));
const clickhouseService_1 = require("../clickhouseService");
const backfillJobStore_1 = require("./backfillJobStore");
const backfillScope_1 = require("./backfillScope");
const backfillRetry_1 = require("./backfillRetry");
const denormalize_1 = require("./denormalize");
const parity_1 = require("./parity");
const types_1 = require("./types");
function useJsonCheckpointFallback() {
    return process.env.EVAL_BACKFILL_JSON_CHECKPOINT === '1';
}
const DEFAULT_BATCH_SIZE = 500;
const MAX_BATCH_SIZE = 2000;
const DEFAULT_MAX_BATCHES = 50000;
const DEFAULT_CHECKPOINT_EVERY_BATCHES = 5;
const MAX_CONCURRENCY = 8;
function resolveBatchSize(explicit) {
    const fromEnv = Number(process.env.EVAL_BACKFILL_BATCH_SIZE);
    const raw = explicit ??
        (Number.isFinite(fromEnv) && fromEnv > 0 ? fromEnv : DEFAULT_BATCH_SIZE);
    return Math.min(MAX_BATCH_SIZE, Math.max(1, Math.floor(raw)));
}
function resolveConcurrency(explicit) {
    const fromEnv = Number(process.env.EVAL_BACKFILL_CONCURRENCY);
    const raw = explicit ??
        (Number.isFinite(fromEnv) && fromEnv > 0 ? fromEnv : 1);
    return Math.min(MAX_CONCURRENCY, Math.max(1, Math.floor(raw)));
}
function resolveCheckpointEveryBatches(explicit) {
    const fromEnv = Number(process.env.EVAL_BACKFILL_CHECKPOINT_EVERY);
    const raw = explicit ??
        (Number.isFinite(fromEnv) && fromEnv > 0 ? fromEnv : DEFAULT_CHECKPOINT_EVERY_BATCHES);
    return Math.max(1, Math.floor(raw));
}
function shardCheckpointPath(basePath, shardIndex) {
    return basePath.replace(/\.json$/i, `.shard-${shardIndex}.json`);
}
function resolveCheckpointPath(targets, override) {
    if (override) {
        return override;
    }
    const fromEnv = process.env.EVAL_BACKFILL_CHECKPOINT_PATH?.trim();
    if (fromEnv) {
        return fromEnv;
    }
    if (targets === 'ch') {
        return '.eval-denorm-backfill-ch.checkpoint.json';
    }
    if (targets === 'pg') {
        return '.eval-denorm-backfill-pg.checkpoint.json';
    }
    return '.eval-denorm-backfill.checkpoint.json';
}
function resolveReadSources(targets, explicit) {
    if (explicit) {
        return [explicit];
    }
    if (targets === 'ch') {
        return ['clickhouse'];
    }
    if (targets === 'pg') {
        return ['postgres'];
    }
    return ['postgres', 'clickhouse'];
}
function normalizeRows(result) {
    if (Array.isArray(result)) {
        return result;
    }
    if (result !== null &&
        typeof result === 'object' &&
        'rows' in result &&
        Array.isArray(result.rows)) {
        return result.rows;
    }
    return [];
}
function loadCheckpoint(path) {
    if (!(0, node_fs_1.existsSync)(path)) {
        return {
            pgAnalyticsInserted: 0,
            pgComponentsInserted: 0,
            chAnalyticsInserted: 0,
            chComponentsInserted: 0,
            skipped: 0,
            errors: [],
        };
    }
    return JSON.parse((0, node_fs_1.readFileSync)(path, 'utf8'));
}
function saveCheckpoint(path, checkpoint) {
    (0, node_fs_1.writeFileSync)(path, JSON.stringify(checkpoint, null, 2));
}
async function saveBackfillProgress(params) {
    if (useJsonCheckpointFallback() && params.checkpointPath) {
        saveCheckpoint(params.checkpointPath, params.checkpoint);
    }
    await (0, backfillJobStore_1.persistBackfillJobProgress)(params.jobId, {
        checkpoint: params.checkpoint,
        status: params.status,
        totalRows: params.totalRows,
        batchNumber: params.batchNumber,
        estimatedBatches: params.estimatedBatches,
        elapsedSeconds: params.elapsedSeconds,
        etaSeconds: params.etaSeconds,
        readSource: params.readSource,
        shardStartOffset: params.shardStartOffset,
        shardEndOffset: params.shardEndOffset,
    });
}
function parseJsonField(raw) {
    if (raw == null || raw === '') {
        return undefined;
    }
    if (typeof raw === 'object') {
        return raw;
    }
    if (typeof raw === 'string') {
        try {
            return JSON.parse(raw);
        }
        catch {
            return undefined;
        }
    }
    return undefined;
}
function rowToDenormInput(row) {
    return {
        id: String(row.id ?? ''),
        createdAt: row.created_at,
        evalId: row.eval_id,
        testIdx: Number(row.test_idx ?? 0),
        testCase: parseJsonField(row.test_case) ?? row.test_case,
        gradingResult: parseJsonField(row.grading_result) ?? row.grading_result,
        metadata: parseJsonField(row.metadata) ?? row.metadata,
        failureReason: Number(row.failure_reason ?? 0),
        error: row.error,
        isRedteam: Number(row.is_redteam ?? 0),
        projectId: row.project_id,
        applicationId: row.application_id,
        workspaceId: row.workspace_id,
        traceId: row.trace_id,
        sessionId: row.session_id,
        traceIds: (parseJsonField(row.trace_ids) ?? row.trace_ids),
        trace_ids: (parseJsonField(row.trace_ids) ?? row.trace_ids),
    };
}
function chLit(value) {
    return value.replace(/'/g, "''");
}
/** Stable total order for cursor pagination. */
function buildBackfillOrderByClause(readSource) {
    if (readSource === 'clickhouse') {
        return 'ORDER BY id ASC, created_at ASC, updated_at ASC, latency_ms ASC, test_idx ASC';
    }
    return 'ORDER BY "id" ASC, "created_at" ASC, "updated_at" ASC, "latency_ms" ASC, "test_idx" ASC';
}
function buildBackfillLimitClause(batchSize) {
    const limit = Math.max(1, Math.floor(batchSize));
    return `LIMIT ${limit}`;
}
/** Legacy: OFFSET bootstrap when an old checkpoint has nextOffset but no resumeCursor. */
function buildBackfillLimitOffsetClause(batchSize, offset) {
    const limit = Math.max(1, Math.floor(batchSize));
    const off = Math.max(0, Math.floor(offset));
    return `LIMIT ${limit} OFFSET ${off}`;
}
function toIsoTimestamp(value) {
    if (value instanceof Date) {
        return value.toISOString();
    }
    if (typeof value === 'string' || typeof value === 'number') {
        const d = new Date(value);
        if (!Number.isNaN(d.getTime())) {
            return d.toISOString();
        }
    }
    return new Date(0).toISOString();
}
function rowToResumeCursor(row) {
    return {
        id: String(row.id ?? ''),
        createdAt: toIsoTimestamp(row.created_at),
        updatedAt: toIsoTimestamp(row.updated_at),
        latencyMs: Number(row.latency_ms ?? 0),
        testIdx: Number(row.test_idx ?? 0),
    };
}
function formatTimestampLiteral(readSource, iso) {
    const d = new Date(iso);
    if (Number.isNaN(d.getTime())) {
        return readSource === 'clickhouse'
            ? "toDateTime('1970-01-01 00:00:00')"
            : "'1970-01-01T00:00:00.000Z'::timestamptz";
    }
    if (readSource === 'clickhouse') {
        const formatted = d.toISOString().slice(0, 19).replace('T', ' ');
        return `toDateTime('${formatted}')`;
    }
    const pgIso = d.toISOString().replace(/'/g, "''");
    return `'${pgIso}'::timestamptz`;
}
/** Rows strictly after `cursor` in the backfill sort order. */
function buildBackfillCursorPredicate(readSource, cursor) {
    const latency = Number.isFinite(cursor.latencyMs) ? Math.floor(cursor.latencyMs) : 0;
    const testIdx = Number.isFinite(cursor.testIdx) ? Math.floor(cursor.testIdx) : 0;
    const created = formatTimestampLiteral(readSource, cursor.createdAt);
    const updated = formatTimestampLiteral(readSource, cursor.updatedAt);
    if (readSource === 'clickhouse') {
        const id = `'${chLit(cursor.id)}'`;
        return `(id, created_at, updated_at, latency_ms, test_idx) > (${id}, ${created}, ${updated}, ${latency}, ${testIdx})`;
    }
    const id = `'${String(cursor.id).replace(/'/g, "''")}'`;
    return `("id", "created_at", "updated_at", "latency_ms", "test_idx") > (${id}, ${created}, ${updated}, ${latency}, ${testIdx})`;
}
function resolveResumeCursor(checkpoint) {
    const cursor = checkpoint.resumeCursor;
    if (!cursor || typeof cursor.id !== 'string' || cursor.id === '') {
        return null;
    }
    return cursor;
}
/** Progress row count from checkpoint (not used for fetch when resumeCursor exists). */
function resolveResumeOffset(checkpoint) {
    if (checkpoint.nextOffset != null && Number.isFinite(checkpoint.nextOffset)) {
        return Math.max(0, Math.floor(checkpoint.nextOffset));
    }
    return 0;
}
function formatDuration(seconds) {
    if (!Number.isFinite(seconds) || seconds < 0) {
        return '?';
    }
    if (seconds < 60) {
        return `${Math.round(seconds)}s`;
    }
    if (seconds < 3600) {
        return `${(seconds / 60).toFixed(1)} min`;
    }
    return `${(seconds / 3600).toFixed(1)} h`;
}
function buildBackfillFilterParts(params) {
    const filters = [];
    const createdAtFilter = params.readSource === 'clickhouse'
        ? (0, backfillScope_1.buildClickhouseCreatedAtFilterSql)(params.scope)
        : (0, backfillScope_1.buildCreatedAtFilterSql)(params.scope);
    if (createdAtFilter) {
        filters.push(createdAtFilter);
    }
    if (params.projectId) {
        if (params.readSource === 'clickhouse') {
            filters.push(`project_id = '${chLit(params.projectId)}'`);
        }
        else {
            filters.push(`"project_id" = '${String(params.projectId).replace(/'/g, "''")}'`);
        }
    }
    if (params.applicationId) {
        if (params.readSource === 'clickhouse') {
            filters.push(`application_id = '${chLit(params.applicationId)}'`);
        }
        else {
            filters.push(`"application_id" = '${String(params.applicationId).replace(/'/g, "''")}'`);
        }
    }
    return filters;
}
async function countEvalResultsForBackfill(params) {
    const filters = buildBackfillFilterParts({
        scope: params.scope,
        readSource: params.readSource,
        projectId: params.projectId,
        applicationId: params.applicationId,
    });
    const where = filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : '';
    if (params.readSource === 'clickhouse') {
        const rows = await clickhouseService_1.clickHouseService.query(`SELECT count() AS n FROM eval_results ${where}`);
        return Number(rows?.[0]?.n ?? 0);
    }
    const db = (0, database_1.getDb)();
    const result = await db.execute(drizzle_orm_1.sql.raw(`SELECT count(*)::bigint AS n FROM "eval_results" ${where}`));
    const normalized = normalizeRows(result);
    return Number(normalized[0]?.n ?? 0);
}
function logBackfillProgress(line) {
    console.log(line);
    logger_1.default.info(line);
}
async function recordBatchFailure(checkpoint, jobId, checkpointPath, error, logPrefix) {
    checkpoint.lastFailedAt = new Date().toISOString();
    checkpoint.lastError = error instanceof Error ? error.message : String(error);
    const cursorHint = checkpoint.resumeCursor?.id
        ? `cursor id=${checkpoint.resumeCursor.id}`
        : `offset=${checkpoint.nextOffset ?? 0}`;
    checkpoint.errors.push(`batch failed at ${cursorHint}: ${checkpoint.lastError}`);
    await saveBackfillProgress({
        jobId,
        checkpoint,
        checkpointPath,
        status: 'failed',
    });
    logBackfillProgress(`${logPrefix} failed after retries; backfill ${jobId} saved (${cursorHint}) — re-run with --resume --backfill-id ${jobId}`);
}
function clearBatchFailure(checkpoint) {
    checkpoint.lastFailedAt = null;
    checkpoint.lastError = null;
}
function buildBackfillFetchQuery(params) {
    const filters = buildBackfillFilterParts({
        scope: params.scope,
        readSource: params.readSource,
        projectId: params.projectId,
        applicationId: params.applicationId,
    });
    if (params.afterCursor) {
        filters.push(buildBackfillCursorPredicate(params.readSource, params.afterCursor));
    }
    const where = filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : '';
    const orderBy = buildBackfillOrderByClause(params.readSource);
    const limitClause = params.afterCursor != null
        ? buildBackfillLimitClause(params.batchSize)
        : buildBackfillLimitOffsetClause(params.batchSize, params.legacyOffset ?? 0);
    if (params.readSource === 'clickhouse') {
        return `
    SELECT
      id,
      created_at,
      updated_at,
      latency_ms,
      eval_id,
      test_idx,
      test_case,
      grading_result,
      metadata,
      failure_reason,
      error,
      is_redteam,
      coalesce(nullIf(project_id, ''), nullIf(projectId, '')) AS project_id,
      application_id,
      coalesce(nullIf(trace_id, ''), nullIf(traceId, '')) AS trace_id,
      coalesce(nullIf(sessionId, ''), '') AS session_id
    FROM eval_results
    ${where}
    ${orderBy}
    ${limitClause}
  `;
    }
    return `
    SELECT
      id, created_at, updated_at, latency_ms, eval_id, test_idx, test_case, grading_result, metadata,
      failure_reason, error, is_redteam, project_id, application_id, workspace_id,
      trace_id, session_id, trace_ids
    FROM "eval_results"
    ${where}
    ${orderBy}
    ${limitClause}
  `;
}
async function fetchEvalResultsBatch(params) {
    const query = buildBackfillFetchQuery(params);
    if (params.readSource === 'clickhouse') {
        return (await clickhouseService_1.clickHouseService.query(query)) ?? [];
    }
    const db = (0, database_1.getDb)();
    return normalizeRows(await db.execute(drizzle_orm_1.sql.raw(query)));
}
function computeOffsetShardRanges(totalRows, shardCount) {
    if (shardCount <= 1 || totalRows <= 0) {
        return [{ startOffset: 0, endOffset: Math.max(0, totalRows) }];
    }
    const shards = [];
    for (let i = 0; i < shardCount; i++) {
        const startOffset = Math.floor((totalRows * i) / shardCount);
        const endOffset = Math.floor((totalRows * (i + 1)) / shardCount);
        shards.push({ startOffset, endOffset });
    }
    return shards;
}
function mergeShardCheckpoints(shards) {
    const merged = {
        pgAnalyticsInserted: 0,
        pgComponentsInserted: 0,
        chAnalyticsInserted: 0,
        chComponentsInserted: 0,
        skipped: 0,
        errors: [],
    };
    for (const shard of shards) {
        merged.pgAnalyticsInserted += shard.pgAnalyticsInserted;
        merged.pgComponentsInserted += shard.pgComponentsInserted;
        merged.chAnalyticsInserted += shard.chAnalyticsInserted;
        merged.chComponentsInserted += shard.chComponentsInserted;
        merged.skipped += shard.skipped;
        merged.errors.push(...shard.errors);
        if (shard.scopeCreatedAfter !== undefined) {
            merged.scopeCreatedAfter = shard.scopeCreatedAfter;
        }
        if (shard.scopeReason) {
            merged.scopeReason = shard.scopeReason;
        }
        if (shard.batchSize) {
            merged.batchSize = shard.batchSize;
        }
        if (shard.readSource) {
            merged.readSource = shard.readSource;
        }
    }
    return merged;
}
async function backfillFromReadSource(params) {
    const { readSource, scope, options, checkpoint, backfillJobId, checkpointPath, writePg, writeCh, retryConfig } = params;
    const batchSize = resolveBatchSize(options.batchSize);
    const maxBatches = Math.max(1, options.maxBatches ?? DEFAULT_MAX_BATCHES);
    const checkpointEvery = resolveCheckpointEveryBatches(options.checkpointEveryBatches);
    const shardStartOffset = params.shardStartOffset ?? 0;
    const shardEndOffset = params.shardEndOffset;
    let afterCursor = options.resume ? resolveResumeCursor(checkpoint) : null;
    let legacyOffsetBootstrap = 0;
    if (!afterCursor) {
        if (params.legacyOffsetBootstrap != null) {
            legacyOffsetBootstrap = Math.max(0, Math.floor(params.legacyOffsetBootstrap));
        }
        else if (options.resume) {
            legacyOffsetBootstrap = Math.max(0, resolveResumeOffset(checkpoint));
        }
        else {
            legacyOffsetBootstrap = Math.max(0, shardStartOffset);
        }
    }
    let rowsProcessed = options.resume && checkpoint.nextOffset != null
        ? resolveResumeOffset(checkpoint)
        : shardStartOffset;
    const logPrefix = params.shardLabel ? `backfill [${readSource}] ${params.shardLabel}` : `backfill [${readSource}]`;
    const skipPostgres = !writePg || readSource !== 'postgres';
    const skipClickHouse = !writeCh || readSource !== 'clickhouse';
    if (skipPostgres && skipClickHouse) {
        return;
    }
    const runStartedAt = Date.now();
    const rowsAlreadyDone = Math.max(0, rowsProcessed - shardStartOffset);
    if (options.resume && checkpoint.lastFailedAt) {
        logBackfillProgress(`${logPrefix} resume after failure at ${checkpoint.lastFailedAt}: ${checkpoint.lastError ?? 'unknown error'}`);
    }
    const retryBatch = (label, operation) => (0, backfillRetry_1.retryWithExponentialBackoff)(operation, {
        label: `${logPrefix} ${label}`,
        config: retryConfig,
        onRetry: (attempt, delayMs, error) => {
            logBackfillProgress(`${logPrefix} retry ${label} attempt ${attempt} in ${delayMs}ms: ${error instanceof Error ? error.message : String(error)}`);
        },
    });
    const totalRows = shardEndOffset != null
        ? Math.max(0, shardEndOffset - shardStartOffset)
        : await retryBatch('count', () => countEvalResultsForBackfill({
            readSource,
            scope,
            projectId: options.projectId,
            applicationId: options.applicationId,
        }));
    const estimatedBatches = totalRows > 0 ? Math.ceil(totalRows / batchSize) : 0;
    const rowsRemaining = Math.max(0, totalRows - rowsAlreadyDone);
    const batchesRemaining = rowsRemaining > 0 ? Math.ceil(rowsRemaining / batchSize) : estimatedBatches;
    logBackfillProgress([
        `${logPrefix} starting`,
        `scope: ${scope.reason}`,
        `eval_results in scope: ${totalRows.toLocaleString()} rows`,
        `batch size: ${batchSize}`,
        `~${estimatedBatches.toLocaleString()} batches total`,
        rowsAlreadyDone > 0
            ? `resume: ${rowsAlreadyDone.toLocaleString()} rows done${afterCursor ? ` (cursor id=${afterCursor.id})` : legacyOffsetBootstrap > 0 ? ` (offset bootstrap ${legacyOffsetBootstrap.toLocaleString()})` : ''} (~${batchesRemaining.toLocaleString()} batches left)`
            : `estimate: ~${formatDuration(batchesRemaining * 0.5)}–${formatDuration(batchesRemaining * 1.5)} (0.5–1.5s/batch ballpark)`,
    ].join(' | '));
    let batchNumber = Math.floor(rowsAlreadyDone / batchSize);
    const fetchBatch = () => {
        if (shardEndOffset != null && rowsProcessed >= shardEndOffset) {
            return Promise.resolve([]);
        }
        const remainingInShard = shardEndOffset != null ? Math.max(0, shardEndOffset - rowsProcessed) : batchSize;
        const effectiveBatchSize = Math.min(batchSize, remainingInShard);
        if (effectiveBatchSize <= 0) {
            return Promise.resolve([]);
        }
        return fetchEvalResultsBatch({
            readSource,
            scope,
            batchSize: effectiveBatchSize,
            projectId: options.projectId,
            applicationId: options.applicationId,
            afterCursor,
            legacyOffset: afterCursor ? undefined : legacyOffsetBootstrap,
        });
    };
    const fetchBatchWithRetry = () => retryBatch('fetch', fetchBatch);
    let pendingFetch = fetchBatchWithRetry();
    for (let batch = 0; batch < maxBatches; batch++) {
        try {
            const evalRows = await (pendingFetch ?? fetchBatchWithRetry());
            pendingFetch = null;
            if (evalRows.length === 0) {
                break;
            }
            const outputs = [];
            for (const row of evalRows) {
                try {
                    outputs.push((0, denormalize_1.denormalizeEvalResult)(rowToDenormInput(row), types_1.DENORM_VERSION_BACKFILL));
                }
                catch (err) {
                    checkpoint.skipped += 1;
                    checkpoint.errors.push(`skip ${row.id}: ${String(err)}`);
                }
            }
            const componentCount = outputs.reduce((n, o) => n + o.components.length, 0);
            if (options.dryRun) {
                if (!skipPostgres) {
                    checkpoint.pgAnalyticsInserted += outputs.length;
                    checkpoint.pgComponentsInserted += componentCount;
                }
                if (!skipClickHouse) {
                    checkpoint.chAnalyticsInserted += outputs.length;
                    checkpoint.chComponentsInserted += componentCount;
                }
            }
            else {
                const prev = process.env.EVAL_DENORM_WRITES;
                process.env.EVAL_DENORM_WRITES = '1';
                try {
                    const { persistDenormOutputsBatch } = await import('./insertEvalDenormArtifacts.js');
                    await retryBatch('persist', () => persistDenormOutputsBatch(outputs, {
                        skipPostgres,
                        skipClickHouse,
                        rethrowOnError: true,
                    }));
                    if (!skipPostgres) {
                        checkpoint.pgAnalyticsInserted += outputs.length;
                        checkpoint.pgComponentsInserted += componentCount;
                    }
                    if (!skipClickHouse) {
                        checkpoint.chAnalyticsInserted += outputs.length;
                        checkpoint.chComponentsInserted += componentCount;
                    }
                }
                finally {
                    if (prev === undefined) {
                        delete process.env.EVAL_DENORM_WRITES;
                    }
                    else {
                        process.env.EVAL_DENORM_WRITES = prev;
                    }
                }
            }
            const lastRow = evalRows[evalRows.length - 1];
            if (lastRow) {
                afterCursor = rowToResumeCursor(lastRow);
                checkpoint.resumeCursor = afterCursor;
                legacyOffsetBootstrap = 0;
            }
            rowsProcessed += evalRows.length;
            checkpoint.nextOffset = rowsProcessed;
            checkpoint.readSource = readSource;
            clearBatchFailure(checkpoint);
            batchNumber += 1;
            const rowsDoneInShard = Math.max(0, rowsProcessed - shardStartOffset);
            const pct = totalRows > 0 ? Math.min(100, (rowsDoneInShard / totalRows) * 100) : null;
            const elapsedSec = (Date.now() - runStartedAt) / 1000;
            const etaSec = totalRows > 0 && rowsDoneInShard > rowsAlreadyDone
                ? (elapsedSec / (rowsDoneInShard - rowsAlreadyDone)) *
                    Math.max(0, totalRows - rowsDoneInShard)
                : null;
            const isLastBatch = evalRows.length < batchSize;
            if (isLastBatch || batchNumber % checkpointEvery === 0) {
                await saveBackfillProgress({
                    jobId: backfillJobId,
                    checkpoint,
                    checkpointPath,
                    totalRows,
                    batchNumber,
                    estimatedBatches,
                    elapsedSeconds: elapsedSec,
                    etaSeconds: etaSec ?? undefined,
                    readSource,
                    shardStartOffset: params.shardStartOffset,
                    shardEndOffset: params.shardEndOffset,
                });
            }
            const progressParts = [
                `${logPrefix} batch ${batchNumber}/${estimatedBatches > 0 ? estimatedBatches : '?'}`,
                pct != null ? `${pct.toFixed(1)}%` : null,
                `rows ${rowsDoneInShard.toLocaleString()}/${totalRows > 0 ? totalRows.toLocaleString() : '?'}`,
                `this batch ${evalRows.length}`,
                afterCursor ? `cursor ${afterCursor.id}` : `offset ${rowsProcessed.toLocaleString()}`,
                `CH ${checkpoint.chAnalyticsInserted.toLocaleString()} / PG ${checkpoint.pgAnalyticsInserted.toLocaleString()} analytics`,
                etaSec != null && rowsDoneInShard < totalRows ? `ETA ~${formatDuration(etaSec)}` : null,
                rowsDoneInShard >= totalRows && totalRows > 0 ? 'finishing' : null,
            ].filter(Boolean);
            logBackfillProgress(progressParts.join(' | '));
            const reachedShardEnd = shardEndOffset != null && rowsProcessed >= shardEndOffset;
            if (evalRows.length < batchSize || reachedShardEnd) {
                await (0, backfillJobStore_1.markBackfillJobCompleted)(backfillJobId, checkpoint, {
                    totalRows,
                    batchNumber,
                    estimatedBatches,
                    elapsedSeconds: elapsedSec,
                    readSource,
                    shardStartOffset: params.shardStartOffset,
                    shardEndOffset: params.shardEndOffset,
                });
                if (useJsonCheckpointFallback() && checkpointPath) {
                    saveCheckpoint(checkpointPath, checkpoint);
                }
                logBackfillProgress(`${logPrefix} complete: ${rowsDoneInShard.toLocaleString()} eval_results in ${batchNumber} batches (${formatDuration(elapsedSec)}) | backfill ${backfillJobId}`);
                break;
            }
            pendingFetch = fetchBatchWithRetry();
        }
        catch (error) {
            pendingFetch = null;
            await recordBatchFailure(checkpoint, backfillJobId, checkpointPath, error, logPrefix);
            throw error;
        }
    }
}
async function backfillEvalResultsDenorm(options) {
    const batchSize = resolveBatchSize(options?.batchSize);
    const concurrency = resolveConcurrency(options?.concurrency);
    const retryConfig = (0, backfillRetry_1.resolveBackfillRetryConfig)({
        maxRetries: options?.maxRetries,
        initialDelayMs: options?.retryInitialDelayMs,
        maxDelayMs: options?.retryMaxDelayMs,
        backoffMultiplier: options?.retryBackoffMultiplier,
    });
    const targets = options?.targets ?? 'both';
    const writePg = targets === 'pg' || targets === 'both';
    const writeCh = targets === 'ch' || targets === 'both';
    const pipeline = (0, backfillJobStore_1.pipelineFromTargets)(targets);
    const jsonCheckpointPath = useJsonCheckpointFallback()
        ? resolveCheckpointPath(targets, options?.checkpointPath)
        : undefined;
    const readSources = resolveReadSources(targets, options?.readSource);
    const scopeFromOptions = (0, backfillScope_1.resolveEvalDenormBackfillScope)({
        sinceDays: options?.sinceDays,
        allHistory: options?.allHistory,
        createdAfter: options?.createdAfter,
        createdBefore: options?.createdBefore,
    });
    let primaryBackfillId = options?.backfillId ?? '';
    let checkpoint = {
        pgAnalyticsInserted: 0,
        pgComponentsInserted: 0,
        chAnalyticsInserted: 0,
        chComponentsInserted: 0,
        skipped: 0,
        errors: [],
    };
    if (options?.resume) {
        if (!options.backfillId) {
            throw new Error('resume requires backfillId (Postgres backfills table)');
        }
        const loaded = await (0, backfillJobStore_1.loadBackfillJobForResume)(options.backfillId);
        if (!loaded) {
            throw new Error(`backfill job not found: ${options.backfillId}`);
        }
        primaryBackfillId = loaded.row.id;
        checkpoint = loaded.checkpoint;
        if (useJsonCheckpointFallback() && jsonCheckpointPath && (0, node_fs_1.existsSync)(jsonCheckpointPath)) {
            checkpoint = { ...checkpoint, ...loadCheckpoint(jsonCheckpointPath) };
        }
    }
    else if (options?.backfillId) {
        primaryBackfillId = options.backfillId;
    }
    let scope = scopeFromOptions;
    if (options?.resume && checkpoint.scopeCreatedAfter !== undefined) {
        const resumedCreatedBefore = checkpoint.scopeCreatedBefore != null && checkpoint.scopeCreatedBefore !== ''
            ? new Date(checkpoint.scopeCreatedBefore)
            : null;
        if (checkpoint.scopeCreatedAfter === null) {
            scope = {
                createdAfter: null,
                createdBefore: resumedCreatedBefore,
                sinceDays: null,
                allHistory: true,
                reason: checkpoint.scopeReason ?? 'resumed: all history',
            };
        }
        else {
            scope = {
                createdAfter: new Date(checkpoint.scopeCreatedAfter),
                createdBefore: resumedCreatedBefore,
                sinceDays: null,
                allHistory: false,
                reason: checkpoint.scopeReason ?? 'resumed checkpoint scope',
            };
        }
    }
    else {
        checkpoint.scopeCreatedAfter = (0, backfillScope_1.scopeCreatedAfterIso)(scope);
        checkpoint.scopeCreatedBefore = (0, backfillScope_1.scopeCreatedBeforeIso)(scope);
        checkpoint.scopeReason = scope.reason;
        checkpoint.batchSize = batchSize;
    }
    if (options?.resume && checkpoint.readSource && !readSources.includes(checkpoint.readSource)) {
        logger_1.default.warn(`Resuming checkpoint from readSource=${checkpoint.readSource}; current run uses ${readSources.join(',')}`);
    }
    for (const readSource of readSources) {
        const resumeThisSource = options?.resume && checkpoint.readSource === readSource;
        const totalForSource = await (0, backfillRetry_1.retryWithExponentialBackoff)(() => countEvalResultsForBackfill({
            readSource,
            scope,
            projectId: options?.projectId,
            applicationId: options?.applicationId,
        }), { label: `backfill [${readSource}] count`, config: retryConfig });
        if (concurrency > 1 && !options?.dryRun) {
            const shards = computeOffsetShardRanges(totalForSource, concurrency);
            if (!primaryBackfillId) {
                primaryBackfillId = await (0, backfillJobStore_1.createBackfillJob)({
                    pipeline,
                    targets,
                    options: options ?? {},
                    readSource,
                    batchSize,
                    scopeCreatedAfter: checkpoint.scopeCreatedAfter,
                    scopeCreatedBefore: checkpoint.scopeCreatedBefore,
                    scopeReason: checkpoint.scopeReason,
                    dryRun: options?.dryRun,
                });
                logBackfillProgress(`backfill job ${primaryBackfillId} (parent, ${shards.length} shards)`);
            }
            const existingChildren = options?.resume && primaryBackfillId
                ? await (0, backfillJobStore_1.listChildBackfillJobs)(primaryBackfillId)
                : [];
            const shardCheckpoints = await Promise.all(shards.map(async (shard, shardIndex) => {
                let shardJobId = existingChildren.find((c) => c.shardIndex === shardIndex)?.id;
                let shardCheckpoint;
                if (shardJobId) {
                    const row = await (0, backfillJobStore_1.getBackfillJobById)(shardJobId);
                    shardCheckpoint = row ? (0, backfillJobStore_1.backfillRowToCheckpoint)(row) : { ...checkpoint };
                }
                else {
                    shardJobId = await (0, backfillJobStore_1.createBackfillJob)({
                        pipeline,
                        targets,
                        options: options ?? {},
                        readSource,
                        parentId: primaryBackfillId,
                        shardIndex,
                        shardStartOffset: shard.startOffset,
                        shardEndOffset: shard.endOffset,
                        batchSize,
                        scopeCreatedAfter: checkpoint.scopeCreatedAfter,
                        scopeCreatedBefore: checkpoint.scopeCreatedBefore,
                        scopeReason: checkpoint.scopeReason,
                        dryRun: options?.dryRun,
                    });
                    shardCheckpoint = {
                        nextOffset: shard.startOffset,
                        pgAnalyticsInserted: 0,
                        pgComponentsInserted: 0,
                        chAnalyticsInserted: 0,
                        chComponentsInserted: 0,
                        skipped: 0,
                        errors: [],
                        scopeCreatedAfter: checkpoint.scopeCreatedAfter,
                        scopeCreatedBefore: checkpoint.scopeCreatedBefore,
                        scopeReason: checkpoint.scopeReason,
                        batchSize,
                    };
                }
                const shardPath = jsonCheckpointPath
                    ? shardCheckpointPath(jsonCheckpointPath, shardIndex)
                    : undefined;
                const shardLegacyBootstrap = resumeThisSource && resolveResumeCursor(shardCheckpoint)
                    ? undefined
                    : shard.startOffset;
                await backfillFromReadSource({
                    readSource,
                    scope,
                    options: { ...(options ?? {}), resume: Boolean(shardJobId && resumeThisSource) },
                    checkpoint: shardCheckpoint,
                    backfillJobId: shardJobId,
                    checkpointPath: shardPath,
                    writePg,
                    writeCh,
                    retryConfig,
                    legacyOffsetBootstrap: shardLegacyBootstrap,
                    shardStartOffset: shard.startOffset,
                    shardEndOffset: shard.endOffset,
                    shardLabel: `shard ${shardIndex + 1}/${shards.length}`,
                });
                const row = await (0, backfillJobStore_1.getBackfillJobById)(shardJobId);
                return row ? (0, backfillJobStore_1.backfillRowToCheckpoint)(row) : shardCheckpoint;
            }));
            const merged = mergeShardCheckpoints(shardCheckpoints);
            checkpoint.pgAnalyticsInserted = merged.pgAnalyticsInserted;
            checkpoint.pgComponentsInserted = merged.pgComponentsInserted;
            checkpoint.chAnalyticsInserted = merged.chAnalyticsInserted;
            checkpoint.chComponentsInserted = merged.chComponentsInserted;
            checkpoint.skipped = merged.skipped;
            checkpoint.errors = merged.errors;
            await (0, backfillJobStore_1.aggregateParentBackfillJob)(primaryBackfillId);
        }
        else {
            if (!primaryBackfillId) {
                primaryBackfillId = await (0, backfillJobStore_1.createBackfillJob)({
                    pipeline,
                    targets,
                    options: options ?? {},
                    readSource,
                    batchSize,
                    scopeCreatedAfter: checkpoint.scopeCreatedAfter,
                    scopeCreatedBefore: checkpoint.scopeCreatedBefore,
                    scopeReason: checkpoint.scopeReason,
                    dryRun: options?.dryRun,
                });
                logBackfillProgress(`backfill job ${primaryBackfillId} started`);
            }
            await backfillFromReadSource({
                readSource,
                scope,
                options: options ?? {},
                checkpoint,
                backfillJobId: primaryBackfillId,
                checkpointPath: jsonCheckpointPath,
                writePg,
                writeCh,
                retryConfig,
            });
            const row = await (0, backfillJobStore_1.getBackfillJobById)(primaryBackfillId);
            if (row) {
                checkpoint = (0, backfillJobStore_1.backfillRowToCheckpoint)(row);
            }
        }
    }
    if (options?.paritySample && options.paritySample > 0 && !options.dryRun) {
        const createdAtFilter = (0, backfillScope_1.buildCreatedAtFilterSql)(scope);
        const parityWhere = createdAtFilter ? `WHERE ${createdAtFilter}` : '';
        const db = (0, database_1.getDb)();
        const sampleQuery = `
      SELECT id, created_at, eval_id, test_idx, test_case, grading_result, metadata,
        failure_reason, error, is_redteam, project_id, application_id, workspace_id, trace_id, session_id
      FROM "eval_results"
      ${parityWhere}
      ORDER BY random()
      LIMIT ${Math.min(500, options.paritySample)}
    `;
        const sampleRows = normalizeRows(await db.execute(drizzle_orm_1.sql.raw(sampleQuery)));
        const parityResults = sampleRows.map((r) => (0, parity_1.compareDenormToLegacyRow)(rowToDenormInput(r)));
        const summary = (0, parity_1.summarizeParity)(parityResults);
        (0, node_fs_1.writeFileSync)('backfill_parity_report.json', JSON.stringify({ summary, mismatches: parityResults.flatMap((r) => r.mismatches) }, null, 2));
        logger_1.default.info(`Parity sample: ${summary.failed}/${summary.total} failed`);
    }
    return { ...checkpoint, backfillId: primaryBackfillId };
}
//# sourceMappingURL=backfillEvalResultsDenorm.js.map