/**
 * Pipeline step: Run Promptfoo evaluation.
 *
 * Uses ctx.evalRunner (the EvalRunner port) for the actual Promptfoo
 * invocation. Builds a clean env object for the subprocess instead of
 * polluting global process.env.
 */
import { existsSync, mkdirSync, writeFileSync } from "fs";
import { resolve } from "path";
import { emitPerEntryEvalResults } from "../../pipeline/emit-eval-results.js";
import { emitSymbolPreflight } from "../../pipeline/preflight/emit-symbol-preflight.js";
import { loadPackageSurface } from "../../pipeline/preflight/load-package-surface.js";
import { AccumulatingArtifactWriter } from "../../artifact-capture/accumulating-artifact-writer.js";
import { FanoutArtifactWriter } from "../../artifact-capture/fanout-artifact-writer.js";
import { InstrumentedArtifactWriter } from "../../artifact-capture/instrumented-artifact-writer.js";
import { getStepInputPaths } from "../../pipeline/cache.js";
import { buildCacheContext } from "../cache-context.js";
import { excludeContextEmptyTasks } from "./context-empty-filter.js";
import { pruneToResolvableRefs, remapToCacheHitRefs, } from "../../pipeline/cache-hit-restore.js";
import { checkCanonicalContextsExist, checkGeneratedConfigsExist, checkResultsExist, } from "../../pipeline/checks.js";
import { computeEvalFingerprint } from "../../pipeline/eval-fingerprint.js";
import { loadGraderModel } from "../../pipeline/grader-api.js";
import { buildFilterFlags, configFileForMode, resultsFileForMode, scanResultsForErrors, } from "../../pipeline/eval-constants.js";
export class RunEvalStep {
    mode;
    name;
    constructor(mode) {
        this.mode = mode;
        this.name = `eval-${mode}`;
    }
    check() {
        return [];
    }
    async execute(ctx, state) {
        if (ctx.config.skipEval) {
            return { status: "skipped", reason: "--no-eval" };
        }
        const start = Date.now();
        const { rootDir, debug, concurrency, noCache } = ctx.config;
        // Precondition: config file exists
        const configIssues = checkGeneratedConfigsExist(rootDir, this.mode);
        const configErrors = configIssues.filter((i) => i.severity === "error");
        if (configErrors.length > 0) {
            return {
                durationMs: Date.now() - start,
                error: `Config files missing: ${configErrors.map((e) => e.message).join("; ")}`,
                status: "failed",
            };
        }
        // Load the task set once and reuse it for both the literacy precondition
        // check and the fingerprint. Mirrors the area/task filter applied by
        // fetch-docs so we only see tasks that were actually fetched.
        const filter = ctx.config.areas ||
            ctx.config.tasks ||
            ctx.config.tags ||
            ctx.config.injectContextDocs?.length
            ? {
                ...(ctx.config.areas ? { areas: ctx.config.areas } : {}),
                ...(ctx.config.tasks ? { taskIds: ctx.config.tasks } : {}),
                ...(ctx.config.tags ? { tags: ctx.config.tags } : {}),
                ...(ctx.config.injectContextDocs?.length
                    ? { injectContextDocs: ctx.config.injectContextDocs }
                    : {}),
            }
            : undefined;
        let tasks = await ctx.taskSource.loadTasks(filter);
        // Release auto-scope: narrow to affected tasks (mirrors GenerateConfigsStep)
        if (state.releaseAutoScope && !ctx.config.noAutoScope) {
            const scopedIds = new Set(state.releaseAutoScope.affectedTaskIds);
            tasks = tasks.filter((t) => scopedIds.has(t.id));
        }
        // Exclude tasks FetchDocsStep skipped for empty canonical context, so the
        // precondition below doesn't re-abort the run on a task that was already
        // dropped (mirrors GenerateConfigsStep).
        tasks = excludeContextEmptyTasks(tasks, state).kept;
        // Precondition: canonical context files exist for filtered tasks.
        // Only applies to literacy mode — other modes don't use canonical doc contexts.
        if (this.mode === "literacy") {
            // Only check context files for tasks that have canonical docs.
            // Tasks without canonical docs are skipped by FetchDocsStep (they
            // have no docs to fetch), so no context file is written for them.
            // The generated Promptfoo config still includes their "without-docs"
            // variant (testing model knowledge alone), which doesn't need a
            // context file.
            const tasksWithDocs = tasks.filter((t) => t.mode === "literacy" && (t.context?.docs?.length ?? 0) > 0);
            const taskIds = tasksWithDocs.map((t) => t.id);
            const contextIssues = checkCanonicalContextsExist(rootDir, taskIds);
            const contextErrors = contextIssues.filter((i) => i.severity === "error");
            if (contextErrors.length > 0) {
                return {
                    durationMs: Date.now() - start,
                    error: `Context files missing. Run fetch-docs first. ${contextErrors.map((e) => e.message).join("; ")}`,
                    status: "failed",
                };
            }
        }
        // -----------------------------------------------------------------
        // Compute eval fingerprint (for remote cache + provenance)
        // -----------------------------------------------------------------
        let evalFingerprint;
        if (!debug?.enabled) {
            try {
                evalFingerprint = computeEvalFingerprint({
                    tasks,
                    graderModel: loadGraderModel(rootDir).id,
                    mode: this.mode,
                    rootDir,
                    graderContext: ctx.config.graderContext,
                });
                // Share fingerprint with downstream steps (PublishReportStep).
                // `evalFingerprint` is the last-writer-wins scalar (back-compat); a
                // FULL literacy run's two sub-runs (baseline + agentic) both write it,
                // so only the last (agentic) survives. `evalFingerprints` accumulates
                // EVERY sub-run's fingerprint so the published report can be resolved
                // from any of them — otherwise an unchanged full re-run's baseline
                // sub-run always misses the remote cache (W-persist-subrun-fingerprints).
                // Accumulated before the remote-cache early-return below so cache-hit
                // sub-runs still contribute their fingerprint to the new report.
                state.evalFingerprint = evalFingerprint;
                state.evalFingerprints ??= [];
                if (!state.evalFingerprints.includes(evalFingerprint)) {
                    state.evalFingerprints.push(evalFingerprint);
                }
            }
            catch (err) {
                console.warn(`  ⚠️  Could not compute eval fingerprint: ${err instanceof Error ? err.message : String(err)}`);
            }
        }
        // -----------------------------------------------------------------
        // Remote cache check
        // -----------------------------------------------------------------
        if (evalFingerprint &&
            !noCache &&
            !ctx.config.noRemoteCache &&
            ctx.reportStore) {
            const remoteCacheResult = await checkRemoteCache(evalFingerprint, ctx.reportStore, rootDir);
            if (remoteCacheResult) {
                // Record the cache hit so CalculateScoresStep can skip when all
                // required eval modes were satisfied from the remote cache.
                state.remoteCacheHits ??= new Set();
                state.remoteCacheHits.add(this.mode);
                // Carry forward the share-link backreference for THIS mode only.
                // Pushing every entry from `remoteCacheResult.promptfooUrls`
                // snowballs across the daily perspective cron: each cache-hit
                // run inherits the cached report's full URL list (including
                // other modes and any URLs the cached report had itself
                // accumulated from earlier hits), then layers its own on top.
                // Iterate from the tail to handle pre-fix cached reports that
                // may carry multiple entries for the same mode.
                const cachedUrls = remoteCacheResult.promptfooUrls;
                let inherited;
                if (cachedUrls) {
                    for (let i = cachedUrls.length - 1; i >= 0; i--) {
                        const entry = cachedUrls[i];
                        if (entry?.mode === this.mode) {
                            inherited = entry;
                            break;
                        }
                    }
                }
                if (inherited) {
                    state.promptfooUrls ??= [];
                    state.promptfooUrls.push(inherited);
                }
                // D0040 / W0135 — restore the cached report's artifact manifest into
                // the accumulator so the new run's RunManifest advertises the cached
                // artifacts via cross-run lineage (`sourceRunId`) instead of skipping
                // them entirely. Without this, Studio drill-downs on the new report
                // 404 because per-entry GCS objects were never written under the new
                // runId. Bytes are not duplicated; the original prefix is untouched.
                if (remoteCacheResult.artifactManifest &&
                    remoteCacheResult.sourceRunId &&
                    ctx.artifactWriter instanceof AccumulatingArtifactWriter) {
                    const restored = remapToCacheHitRefs(remoteCacheResult.artifactManifest, { sourceRunId: remoteCacheResult.sourceRunId });
                    // W0350 / D0057 — a degraded source run can advertise per-entry
                    // artifacts (e.g. rawResults) whose objects were never written under
                    // its prefix. Drop those over-claims here, at the restore boundary,
                    // so the new run's manifest advertises only artifacts that resolve —
                    // rather than pushing per-object HEAD checks onto the read side's hot
                    // signing path (AC 3). When no object checker is reachable in the
                    // writer chain (local-only / NoOp / gateway backends), skip the prune
                    // and restore verbatim, preserving prior behavior.
                    const checker = findObjectChecker(ctx.artifactWriter);
                    const { manifest: resolvable, droppedEntries, droppedRefs, } = checker
                        ? await pruneToResolvableRefs(restored, checker)
                        : { manifest: restored, droppedEntries: 0, droppedRefs: 0 };
                    ctx.artifactWriter.injectAccumulated(resolvable);
                    const count = Object.keys(resolvable).length;
                    if (count > 0) {
                        console.log(`  ↪ Restored ${count} artifact ref${count === 1 ? "" : "s"} from run ${remoteCacheResult.sourceRunId}`);
                    }
                    if (droppedEntries > 0 || droppedRefs > 0) {
                        const refsNote = droppedRefs > 0
                            ? ` and ${droppedRefs} ref${droppedRefs === 1 ? "" : "s"}`
                            : "";
                        console.log(`  ⚠️  Dropped ${droppedEntries} unresolvable artifact entr${droppedEntries === 1 ? "y" : "ies"}${refsNote} over-claimed by cache parent ${remoteCacheResult.sourceRunId}`);
                    }
                }
                return {
                    durationMs: Date.now() - start,
                    status: "success",
                    summary: `Skipped (remote cache hit) — reusing report ${remoteCacheResult.reportId} from ${remoteCacheResult.completedAt}`,
                };
            }
        }
        // -----------------------------------------------------------------
        // Build subprocess env explicitly (no global mutation)
        // -----------------------------------------------------------------
        const subprocessEnv = {
            PROMPTFOO_DISABLE_SHARE_EMAIL_REQUEST: "1",
        };
        // Only set env vars that differ from defaults — the subprocess inherits
        // process.env via PromptfooEvalAdapter's { ...process.env, ...config.env }
        if (ctx.config.mode !== "literacy") {
            subprocessEnv.EVAL_MODE = ctx.config.mode;
        }
        if (ctx.config.searchMode !== "open") {
            subprocessEnv.EVAL_SEARCH_MODE = ctx.config.searchMode;
        }
        if (ctx.config.allowedOrigins?.length) {
            subprocessEnv.DOC_ALLOWED_ORIGINS = ctx.config.allowedOrigins.join(",");
        }
        // -----------------------------------------------------------------
        // Execute — use the EvalRunner port
        // -----------------------------------------------------------------
        const configFile = configFileForMode(this.mode);
        const filterFlags = buildFilterFlags(debug);
        const result = await ctx.evalRunner.run({
            concurrency,
            configPath: configFile,
            env: subprocessEnv,
            filterFlags: filterFlags.trim() || undefined,
            maxDurationMs: ctx.config.evalBudgetMs,
        });
        // Check if results were written despite non-zero exit
        if (result.status === "failed") {
            const resultsExist = checkResultsExist(rootDir, resultsFileForMode(this.mode));
            const hasResults = resultsExist.filter((i) => i.severity === "error").length === 0;
            if (!hasResults) {
                return {
                    durationMs: Date.now() - start,
                    error: result.error ?? `promptfoo eval failed (mode: ${this.mode})`,
                    status: "failed",
                };
            }
        }
        // Postcondition: results file exists
        const resultsIssues = checkResultsExist(rootDir, resultsFileForMode(this.mode));
        const resultsErrors = resultsIssues.filter((i) => i.severity === "error");
        if (resultsErrors.length > 0) {
            return {
                durationMs: Date.now() - start,
                error: `Postcondition failed: ${resultsErrors.map((e) => e.message).join("; ")}`,
                status: "failed",
            };
        }
        // Scan results for errors
        const errorSummary = scanResultsForErrors(resolve(rootDir, resultsFileForMode(this.mode)));
        if (errorSummary) {
            console.log();
            console.log(errorSummary);
        }
        // W0050 — decompose the promptfoo aggregate into the per-entry
        // descriptors the W0049 registry expects: rawResults / renderedPrompts
        // per (run, mode, task, model); graderPrompts / graderJudgments per
        // (run, mode, task, model, grader). See pipeline/emit-eval-results.ts.
        // `testOutputs` still flows through uploadTestOutputs() in
        // calculate-scores-step. `traces` ships via agent-observer (out of
        // scope for the promptfoo shape parser — follow-up).
        const resultsPath = resolve(rootDir, resultsFileForMode(this.mode));
        if (existsSync(resultsPath)) {
            await emitPerEntryEvalResults(ctx.artifactWriter, ctx, this.mode, resultsPath);
            // W0198 Phase 4 — deterministic-lane reports per (task, model).
            // Loaded lazily so test contexts that don't wire the manifest /
            // resolver pay nothing; the helper is itself a no-op when its
            // inputs are missing.
            const packageSurface = await loadPackageSurface(rootDir).catch((err) => {
                console.warn(`  ⚠️  W0198 preflight: failed to load package-surface manifest — ${err instanceof Error ? err.message : String(err)}`);
                return undefined;
            });
            const preflight = await emitSymbolPreflight({
                writer: ctx.artifactWriter,
                ctx,
                mode: this.mode,
                resultsPath,
                packageSurface,
                resolver: ctx.packageSurfaceResolver,
            });
            if (preflight.reports.size > 0) {
                if (!state.preflightReports) {
                    state.preflightReports = new Map();
                }
                for (const [k, v] of preflight.reports) {
                    state.preflightReports.set(k, v);
                }
            }
        }
        // Extract Promptfoo share URL from eval results (Step 3b)
        if (ctx.evalRunner.extractShareUrl) {
            const shareUrl = ctx.evalRunner.extractShareUrl(resolve(rootDir, resultsFileForMode(this.mode)));
            if (shareUrl) {
                state.promptfooUrls ??= [];
                state.promptfooUrls.push({ mode: this.mode, url: shareUrl });
            }
        }
        const durationMs = Date.now() - start;
        return {
            durationMs,
            status: "success",
            summary: `Evaluation complete (mode: ${this.mode}${debug?.enabled ? ", debug" : ""})`,
        };
    }
    cacheInputs(ctx) {
        return getStepInputPaths(ctx.config.rootDir, `eval-${this.mode}`);
    }
    cacheContext(ctx) {
        return buildCacheContext(ctx.config);
    }
}
// ---------------------------------------------------------------------------
// Object-checker discovery (D0057 / W0350)
// ---------------------------------------------------------------------------
const FIND_CHECKER_MAX_STEPS = 16;
/**
 * Walk a writer's decorator/fanout chain to feature-detect an
 * `ArtifactObjectChecker`. The composition root wraps the backend in
 * `AccumulatingArtifactWriter` (and optionally `InstrumentedArtifactWriter`)
 * and layers GCS over local via `FanoutArtifactWriter`. Only
 * `GcsArtifactWriter` implements `objectExists`, so on local-only / NoOp /
 * gateway chains this returns null and the cache-hit restore skips pruning.
 * `MAX_STEPS` is a cycle guard against a future decorator self-reference.
 */
function findObjectChecker(writer) {
    const stack = [writer];
    for (let steps = 0; stack.length > 0 && steps < FIND_CHECKER_MAX_STEPS; steps++) {
        const cursor = stack.pop();
        if (!cursor)
            continue;
        if (hasObjectExists(cursor))
            return cursor;
        if (cursor instanceof AccumulatingArtifactWriter)
            stack.push(cursor.inner);
        else if (cursor instanceof InstrumentedArtifactWriter)
            stack.push(cursor.inner);
        else if (cursor instanceof FanoutArtifactWriter)
            stack.push(...cursor.delegates);
    }
    return null;
}
function hasObjectExists(w) {
    return (typeof w.objectExists === "function");
}
// ---------------------------------------------------------------------------
// Remote cache helpers
// ---------------------------------------------------------------------------
async function checkRemoteCache(fingerprint, reportStore, rootDir) {
    try {
        const startQuery = Date.now();
        const cachedReport = (await reportStore.findByFingerprint(fingerprint));
        const queryMs = Date.now() - startQuery;
        if (!cachedReport) {
            console.log(`  ℹ️  Remote cache miss — no report matches fingerprint (${queryMs}ms)`);
            return null;
        }
        const outDir = resolve(rootDir, "results", "latest");
        if (!existsSync(outDir)) {
            mkdirSync(outDir, { recursive: true });
        }
        writeFileSync(resolve(outDir, "score-summary.json"), JSON.stringify(cachedReport.summary, null, 2));
        console.log(`  ✅ Remote cache hit — reusing report ${cachedReport.id} from ${cachedReport.completedAt}`);
        console.log(`  ℹ️  Fingerprint: ${fingerprint.slice(0, 16)}... (${queryMs}ms)`);
        return {
            artifactManifest: cachedReport.artifactManifest,
            completedAt: cachedReport.completedAt,
            promptfooUrls: cachedReport.provenance?.promptfooUrls,
            reportId: cachedReport.id,
            sourceRunId: cachedReport.provenance?.runId,
        };
    }
    catch (err) {
        console.warn(`  ⚠️  Remote cache check failed: ${err instanceof Error ? err.message : String(err)}`);
        return null;
    }
}
