/**
 * @sanity/ailf-core — Domain types
 *
 * Core types for the AI Literacy Framework evaluation pipeline.
 * These define the contracts between pipeline steps, domain services,
 * and port adapters.
 *
 * Extracted from packages/eval/src/pipeline/types.ts during the
 * Ports & Adapters migration (Phase 0c). The original file is now a
 * re-export barrel that preserves backward compatibility.
 */
import type { DocumentRef as _DocumentRef, EvalMode, LiteracyVariant, RunClassification, RunContext } from "../../ailf-shared/index.d.ts";
import type { ArtifactType } from "../artifact-registry.js";
import type { ErrorSummary } from "./storage-schema.js";
import type { ReleaseImpact } from "../ports/doc-fetcher.js";
import type { GeneralizedDocRef } from "./generalized-task.js";
import type { SymbolPreflightReport } from "./symbol-preflight-report.js";
import type { AssociationValues, RunId } from "./branded-ids.js";
import type { GraderJudgment } from "./grader-judgment.js";
import type { ReportValidity } from "./report-validity.js";
export type { ActualScoreEntry, ComponentResult, TestResult, UrlMetadata, } from "./scoring-input.js";
export type { DocumentRef, RunContext, RunTrigger } from "../../ailf-shared/index.d.ts";
export type { AdminAlertConfig, AdminAlertChannel, } from "./admin-alert-config.js";
export { ADMIN_ALERT_CONFIG_TYPE, ADMIN_ALERT_CONFIG_SINGLETON_ID, } from "./admin-alert-config.js";
export type { ErrorSummary, FullyErroredTask, StoredBaseline, StoredReport, StoredRun, StoredTaskResult, StoredTrace, SchemaVersioned, } from "./storage-schema.js";
export { CURRENT_SCHEMA_VERSION, isSchemaVersioned, migrateDocument, } from "./storage-schema.js";
export type { AssertionRegistration, FixtureResolverRegistration, ModeBase, ModeRegistration, PluginManifest, PluginRegistry, PresetDefinition, ReportSinkRegistration, RubricTemplateRegistration, } from "./plugin-registry.js";
export { InMemoryPluginRegistry } from "./plugin-registry.js";
export type { AgentHarnessConfig, AgentHarnessModeConfig, CustomModeConfig, EvalModeConfig, EvalModeType, KnowledgeBaseRef, KnowledgeProbeModeConfig, LiteracyModeConfig, MCPServerConfig, MCPServerModeConfig, ProbeStrategy, SandboxConfig, ToolDef, } from "./eval-mode-config.js";
export { evalModeType } from "./eval-mode-config.js";
export type { DependencyEdge, ResolvedFixture, TaskGraph, TaskNode, } from "./task-graph.js";
export type { VariableDeclaration, VariableEnvelope, VariableProvenance, VariableSource, } from "./variable-envelope.js";
export type { EvalTrace, ToolCallCategory, ToolCallRecord, ToolCallSource, TraceEvent, TraceSpan, TraceTokenUsage, } from "./trace.js";
export type { CompareRequest, CompareScoreSummaryRef, LookupDocRequest, ValidateConfigRequest, ValidateSourceRequest, ValidateTaskAssertionDef, ValidateTaskCanonicalDocRef, ValidateTaskDef, ValidateTaskRequest, } from "./api-requests.js";
export type { AilfEvalWorkflow, AilfEvalWorkflowJob, AilfEvalWorkflowStep, RepoAgenticConfig, RepoArtifactsConfig, RepoConfig, RepoExecutionConfig, RepoOutputConfig, RepoOwnerConfig, RepoProvenanceConfig, RepoPublishConfig, RepoReportStoreConfig, RepoSourceConfig, RepoSourceSetting, RepoTaskSourceConfig, RepoTriggersConfig, ScheduleTriggerConfig, TriggerConfig, TriggerMode, } from "./repo-config.js";
export type { PipelineRequest, PipelineRequestCallback, PipelineRequestCallerExecutor, PipelineRequestCallerGit, PipelineRequestCallerOwner, PipelineRequestDebug, PipelineRequestTaskSource, } from "./pipeline-request.js";
export type { AskJob, AskJobContextEntry, AskJobSlackDelivery, Job, JobBase, JobCallback, JobError, JobExecution, JobProgress, JobStatus, PipelineJob, } from "./job.js";
export type { PackageSurfaceConfig, PackageSurfaceEntry, } from "./package-surface.js";
export type { SymbolPreflightDeduction, SymbolPreflightFinding, SymbolPreflightReport, SymbolPreflightUnresolvedReason, } from "./symbol-preflight-report.js";
export { DEFAULT_PREFLIGHT_CODE_CORRECTNESS_WEIGHT, type PreflightRubricContext, type PreflightScoringConfig, } from "./preflight-scoring.js";
export type { Confidence, ConfidenceDerivation } from "./confidence.js";
export { CONVENTIONAL_DERIVATIONS, isConfidence } from "./confidence.js";
export type { ArtifactId, AssociationAxis, AssociationValues, Brand, EntryKey, Err, FixtureId, IdValidationError, JudgmentId, NewReportId, Ok, ProviderId, PromptId, Result, ResultId, RubricId, RunFingerprint, RunId, SuiteId, TaskId, TaskSlug, TraceId, } from "./branded-ids.js";
export { err, fixtureId, generateJudgmentId, generateRunId, judgmentId, ok, providerId, resultId, runId, suiteId, taskId, traceId, } from "./branded-ids.js";
export type { AgentHarnessTaskDefinition, ContentLakeAuthorableMode, ContentLakeAuthorableTask, CriterionRef, CustomTaskDefinition, GeneralizedAssertionDefinition, GeneralizedDocRef, GeneralizedTaskDefinition, GeneralizedTemplatedAssertion, GeneralizedValueAssertion, IdDocRef, KnowledgeProbeTaskDefinition, LiteracyTaskDefinition, MCPServerTaskDefinition, PromptVars, PathDocRef, PerspectiveDocRef, ReservedPromptVarKey, RubricRef, SlugDocRef, TaskCommonFields, TaskDifficulty, TaskOptions, TaskProviderConfig, TaskStatus, } from "./generalized-task.js";
export type { TaskPublishState, TaskViabilityReason, TaskViabilityResult, } from "./task-viability.js";
export type { ActionSuggestion, AreaSummaryBody, CardMeta, CardType, Diagnosis, DiagnosisCard, DocAttributionSpotlightBody, FailureModeSummaryBody, JudgmentRef, LowConfidenceAttributionBody, NoIssuesBody, RegressionVsBaselineBody, TopRecommendationsBody, VersionedInputs, WeakestAreaBody, } from "./diagnosis.js";
export type { SynthesisCostTelemetry, SynthesisPerCardTelemetry, } from "./synthesis-telemetry.js";
export type { AttributionMeta, DocAttribution, JudgmentAttribution, } from "./attribution.js";
export type { RecommendationApplication, RecommendationApplicationOutcome, } from "./recommendation-application.js";
export type { RecommendationStatusEvent } from "./recommendation-status-event.js";
export type { ReportAcknowledgementEvent } from "./report-acknowledgement-event.js";
export type { WebhookEventEnvelope, WebhookEventReport, } from "./webhook-event.js";
export type { AreaTrend, DigestSummary, TrendDirection } from "./digest.js";
export type { CriterionSubJudgment, DocCitation, DocCitationRole, GraderEmittedJudgment, GraderJudgment, } from "./grader-judgment.js";
export type { LegacyGraderJudgment } from "./legacy-grader-judgment.js";
export type { BaseChannel, ChannelScope, EmailChannel, EventType, KnownEventType, KnownMemberRole, MemberRole, NotificationChannel, NotificationChannelType, NotificationDelivery, SlackChannel, Team, TeamId, TeamMember, TeamRef, TeamSlug, TeamStatus, WebhookChannel, } from "./team.js";
export type { AilfUser, AilfUserPreferences, TeamReference } from "./user.js";
export { isReportValidityStatus, REPORT_VALIDITY_STATUSES, type ReportValidity, type ReportValidityMethod, type ReportValidityStatus, } from "./report-validity.js";
type DocumentRef = _DocumentRef;
/** Aggregated retrieval metrics for a feature area */
export interface AreaRetrievalMetrics {
    area: string;
    avgF1: number;
    avgPrecision: number;
    avgRecall: number;
    /** Unique canonical slugs that were never retrieved across all tasks */
    invisibleDocs: string[];
    /** Number of tasks with retrieval data */
    taskCount: number;
    tasks: TaskRetrievalMetrics[];
}
/** A parsed task-to-canonical-docs mapping */
export interface CanonicalMapping {
    canonical_docs: {
        slug: string;
        reason: string;
    }[];
    description: string;
    id: string;
    reference_solution: string;
}
/** A document in a content release, classified by change type and evaluation tier */
export interface ClassifiedReleaseDocument {
    /** Feature areas whose tasks reference this document (via canonical_docs) */
    affectedAreas: string[];
    /** Whether this is a new, updated, or removed document */
    changeType: "new" | "removed" | "updated";
    /** The document slug */
    slug: string;
    /** For new docs: Tier A (area has tasks) or Tier B (no tasks exist) */
    tier: "A" | "B" | "not-applicable";
    /** Whether this document maps to any existing task's canonical_docs */
    tracked: boolean;
}
/** Coverage audit report */
export interface CoverageAuditReport {
    /** Coverage percentage */
    coveragePercent: number;
    /** Covered features (with tasks) */
    covered: (ProductFeature & {
        actualTaskCount: number;
    })[];
    /** Timestamp of audit */
    generatedAt: string;
    /** Total features in registry */
    totalFeatures: number;
    /** Uncovered features (no tasks), sorted by priority */
    uncovered: ProductFeature[];
}
/** Debug mode options for running a subset of tests */
export interface DebugOptions {
    /** Enable debug mode (run subset of tests) */
    enabled: boolean;
    /** Number of tests to run (default: 2) */
    firstN?: number;
    /** Regex pattern to filter tests by description */
    pattern?: string;
    /** Random sample of N tests */
    sample?: number;
}
export type { EvalMode, LiteracyVariant, RunClassification, } from "../../ailf-shared/index.d.ts";
/** A classified failure mode with confidence level */
export interface FailureMode {
    /** How confident we are in this classification */
    confidence: "high" | "low" | "medium";
    /** The failure mode category */
    mode: FailureModeType;
    /** Source of the classification signal */
    source: "ceiling" | "keyword" | "keyword+ceiling";
}
/** Summary report of failure modes across all judgments */
export interface FailureModeReport {
    /** Per-area failure mode distribution */
    byArea: Record<string, {
        area: string;
        modes: Record<FailureModeType, number>;
        topMode: FailureModeType;
        totalJudgments: number;
    }>;
    /** Percentage of judgments classified as something other than unclassified */
    classificationRate: number;
    /** Judgments that were classified */
    classifiedJudgments: {
        classification: FailureMode;
        judgment: GraderJudgment;
    }[];
    /** Summary counts by failure mode */
    summary: Record<FailureModeType, number>;
    /** Total judgments analyzed */
    totalJudgments: number;
}
/**
 * Failure mode classification for a low-scoring judgment.
 *
 * Open-set string (Plan 03-02 per-dimension taxonomies introduce modes
 * outside the original literacy enum: `false-floor`, `spec-mismatch`,
 * `tool-misuse`, `factual-error`, `hallucination`, etc. — the grader
 * is told these are legal answers via the rubric prompt). The legacy
 * literacy enum survives as `LegacyFailureModeType` for the
 * report-aggregation helpers that need stable bucket ordering and
 * icon tables; consumers that only care about presence/absence treat
 * `FailureModeType` as `string`.
 */
export type FailureModeType = string;
/**
 * Closed enum of the original literacy failure modes — used by the
 * report formatters that iterate buckets in a stable order. Adding to
 * this list is a deliberate extension; modes outside it still flow
 * through the report (per-area `modes` record), just without a
 * pre-allocated bucket in `summary`.
 */
export type LegacyFailureModeType = "api-error" | "incorrect-docs" | "missing-docs" | "model-limitation" | "outdated-docs" | "poor-structure" | "unclassified";
/** Set of canonical legacy modes — exported for report-formatter use. */
export declare const LEGACY_FAILURE_MODES: readonly LegacyFailureModeType[];
/** Type guard for legacy modes. */
export declare function isLegacyFailureMode(mode: string): mode is LegacyFailureModeType;
/** Per-feature-area score breakdown */
export interface FeatureScore {
    /**
     * Actual score — agentic mode, model retrieves docs via tools.
     * Only present when agentic evaluation data is available.
     */
    actualScore?: number;
    /**
     * Assertion pass rate — fraction of structural assertions that passed (0–100).
     * Only present for modes with javascript assertions (agent-harness, agent-task).
     */
    assertionPassRate?: number;
    /**
     * Ceiling score — gold-standard docs injected directly.
     * This is the theoretical maximum score for this area given the current docs.
     * Set to 0 for modes without with/without-docs variants (agent-harness).
     */
    ceilingScore: number;
    codeCorrectness: number;
    /**
     * Generic dimension scores map — all dimensions by kebab-case name (0–100).
     * Includes the three named fields above plus any mode-specific dimensions
     * (e.g., agent-output, tool-usage, assertion-pass-rate).
     * New consumers should read from this map. The named fields are backward-
     * compatible accessors populated from it.
     */
    dimensions?: Record<string, number>;
    docCoverage: number;
    /** Sanity documents used for this feature area's evaluation */
    documents?: DocumentRef[];
    /**
     * Doc Lift — documentation quality contribution (ceiling − floor).
     * Positive when docs help, negative when docs hurt (interference).
     * Set to 0 for modes without with/without-docs variants (agent-harness).
     */
    docLift: number;
    /**
     * Doc quality gap — room for documentation improvement (100 − ceiling).
     * Lower is better.
     * Set to 0 for modes without with/without-docs variants (agent-harness).
     */
    docQualityGap: number;
    feature: string;
    /**
     * Floor score — no docs, training data only.
     * The model's inherent knowledge baseline.
     * Set to 0 for modes without with/without-docs variants (agent-harness).
     */
    floorScore: number;
    /**
     * How this score entry was grouped.
     *   - "feature": by documentation feature area (literacy mode)
     *   - "task": by individual task ID (agent-harness mode)
     *   - "aggregate": single aggregate across all tasks
     * Defaults to "feature" when absent (backward compatibility).
     */
    groupType?: "aggregate" | "feature" | "task";
    /**
     * Infrastructure efficiency — actual / ceiling (0.0–1.0).
     * What fraction of documentation quality reaches agents through discovery?
     * Null when ceiling ≤ 0, negative Doc Lift, or data is missing.
     */
    infrastructureEfficiency?: number | null;
    /**
     * Whether the retrieval gap is inverted — agents that can't find bad docs
     * outperform those that can. Only true when Doc Lift is negative AND
     * actual > ceiling. Signals a documentation quality problem being masked
     * by retrieval failure. See evaluation-ceiling.md.
     */
    invertedRetrievalGap?: boolean;
    /** Model identifier (e.g., "openai:gpt-4o"). Only present in per-model breakdowns. */
    modelId?: string;
    /**
     * Whether Doc Lift is negative — docs hurt performance.
     * When true, documentation is actively interfering with the model's
     * training-data knowledge. See evaluation-ceiling.md.
     */
    negativeDocLift: boolean;
    /**
     * Retrieval gap — discovery/infrastructure loss (ceiling − actual).
     * Positive means agents can't find all the good docs.
     * Negative means agents outperform injected docs (inverted — see above).
     * Only present when both ceiling and agentic data are available.
     */
    retrievalGap?: number;
    taskCompletion: number;
    testCount: number;
    totalCost: number;
    totalScore: number;
}
/** Scoping options to evaluate only specific areas or tasks */
export interface FilterOptions {
    /** Feature areas to include (filename stems, e.g., ["groq", "frameworks"]) */
    areas?: string[];
    /** Include draft-status tasks in addition to active tasks */
    includeDrafts?: boolean;
    /** Tags to include — tasks must have at least one matching tag */
    tags?: string[];
    /** Specific task IDs to include (e.g., ["groq-blog-queries"]) */
    taskIds?: string[];
    /**
     * Doc references the run is scoped to. When set, only tasks whose
     * `context.docs[*]` matches an entry are returned. A bare `string` keeps
     * exact slug-match semantics; a `GeneralizedDocRef` matches on its
     * strongest key (id, then path, then slug) per D0090. Additive — existing
     * `string[]` values are unchanged. Empty array is a no-op.
     */
    changedDocs?: readonly (string | GeneralizedDocRef)[];
    /**
     * Doc refs to append to every returned task's `context.docs[]`
     * (doc-impact eval). Empty/undefined is a no-op.
     */
    injectContextDocs?: readonly GeneralizedDocRef[];
}
/** Full gap analysis report */
export interface GapAnalysisReport {
    /** All identified gaps, sorted by priority */
    gaps: GapEstimate[];
    /** Timestamp of analysis */
    generatedAt: string;
    /** Aggregate potential lift if all gaps were fixed */
    totalPotentialLift: number;
}
/** A single gap with estimated impact if fixed */
export interface GapEstimate {
    /** The tasks affected by this gap */
    affectedTaskIds: string[];
    /** The area this gap affects */
    area: string;
    /** The dimension(s) bottlenecked by this gap */
    bottleneckDimensions: string[];
    /** Confidence in the estimate */
    confidence: "high" | "low" | "medium";
    /** Current score of the bottleneck dimension(s) */
    currentDimensionScores: Record<string, number>;
    /** Estimated composite score lift if this gap is fully closed */
    estimatedLift: number;
    /** The failure mode causing the gap */
    failureMode: FailureModeType;
    /** Priority score (lift × affectedTaskCount) */
    priority: number;
    /** Specific remediation description */
    remediation: string;
}
/** Enriched grader judgment with stored documentation context refs. */
export interface StoredJudgment extends GraderJudgment {
    /**
     * Documentation context the task expected the model to use.
     *
     * Legacy alias `canonicalDocs` may appear on stored reports written
     * before Phase 2 — readers should tolerate both. Writers (the pipeline)
     * always emit `contextDocs`.
     */
    contextDocs?: DocumentRef[];
}
/**
 * Per-test result stored in reports for drill-down and audit.
 *
 * Captures the model's response output, grader reasoning per dimension,
 * and response metadata. One entry per test × model combination.
 * See D0029 and docs/design-docs/score-drill-down.md (Phase 1).
 */
export interface StoredTestResult {
    /** Resolved feature area (from __featureArea or description) */
    area: string;
    /**
     * Documentation context the task expected the model to use.
     * Legacy alias `canonicalDocs` may appear on pre-Phase-2 reports.
     */
    contextDocs?: DocumentRef[];
    /** Weighted composite score (gold variant only) */
    compositeScore?: number;
    /** Per-test cost (USD) */
    cost?: number;
    /** Per-dimension grader scores and reasoning */
    dimensions: {
        /** Rubric dimension: task-completion, code-correctness, doc-coverage */
        dimension: string;
        /** Grader's natural language reasoning */
        reason: string;
        /** Numeric score (0–100, normalized) */
        score: number;
    }[];
    /** Response latency in milliseconds */
    latencyMs?: number;
    /** Model that produced the response */
    modelId: string;
    /**
     * True when the model failed to produce meaningful output (empty response,
     * API error, or refusal). Same semantics as GraderJudgment.outputFailure.
     */
    outputFailure?: boolean;
    /**
     * The model's generated code/response (truncated to 8000 chars).
     * Populated by the scoring step and used by uploadTestOutputs. Stripped
     * from the inline shape after upload (D0030) — the full value lives in
     * the GCS artifact, keyed by `{taskId}::{modelId}`.
     */
    responseOutput?: string;
    /** True when responseOutput was truncated from a longer response */
    responseOutputTruncated?: boolean;
    /** Task description (e.g. "Functions - Webhook handler (gold)") */
    taskId: string;
    /**
     * Token usage breakdown. All fields are optional because Promptfoo's
     * row-level shape varies — cached results typically carry only
     * `{ cached, total }` while fresh results carry `{ prompt, completion,
     * total }`. Populated from `response.tokenUsage` on the raw result; see
     * `extractStoredTestResults` in packages/eval/src/pipeline/calculate-scores.ts.
     */
    tokenUsage?: {
        cached?: number;
        completion?: number;
        prompt?: number;
        total?: number;
    };
    /** "gold" (with docs) or "baseline" (without docs) */
    variant: "baseline" | "gold";
}
/** Grader consistency diagnostics — does not affect scores, reported alongside */
export interface GraderReliability {
    /**
     * Plan 03-03 — count of grader-emission vs ceiling cross-check disagreements.
     *
     * Incremented by the live pipeline when `validateFailureMode(...)` returns
     * `level: "medium"` (the grader's emitted `failureMode` does not agree with
     * the ceiling-decomposition mode). Surfaces calibration drift over time
     * without affecting scores. Optional — undefined when the run did not
     * exercise the failure-mode validator (e.g., grader-consistency-only paths).
     *
     * @see docs/decisions/D0049-shared-confidence-contract.md
     */
    failureModeCalibration?: number;
    /**
     * Plan 03-03 — count of strict-schema parse failures during grader-output
     * extraction. Wired at the parse-fail branch in `extractGraderJudgments`;
     * incremented when `GraderJudgmentSchema.safeParse` rejects a payload and
     * the pipeline drops to the Phase 1 minimal-shape fallback.
     *
     * Plan 03-04 will tighten the strict schema (`.strict()` + GRAD-02 fields
     * required) and graders will emit the structured wire format in earnest;
     * this counter measures pre-hard-fail drift.
     */
    parseFailures?: number;
    /**
     * Phase 4 ATTR-01 — count of grader citations whose `slug` was not
     * in the resolvable-set (`hallucinationCheckedAgainst`).
     * Incremented by `computeJudgmentAttribution(...)` for every
     * citation that fails the hallucination short-circuit (Success
     * Criterion #5). A counter, not a ratio — consumers compute the
     * rate by dividing by total-citations if needed.
     *
     * @see docs/decisions/D0049-shared-confidence-contract.md
     */
    hallucinationCount?: number;
    /** Inter-grader agreement (from multi-grader comparison) — Phase 3 */
    agreement?: {
        /** Models compared against the primary grader */
        comparedModels: string[];
        /** Pearson correlation with each compared model */
        correlations: Record<string, number>;
        /** Systematic bias (mean signed difference) per model */
        bias: Record<string, number>;
        /** Mean absolute difference per model */
        meanAbsDiff: Record<string, number>;
        /** Recommendation per candidate: comparable, divergent, or strongly-divergent */
        recommendations: Record<string, string>;
    };
    /** Intra-grader consistency (from replicated grading runs) — Phase 1 */
    consistency?: {
        /** Average standard deviation across all grading judgments */
        avgStdDev: number;
        /** Max standard deviation observed (worst-case noise) */
        maxStdDev: number;
        /** Average score range (max - min) across judgments */
        avgRange: number;
        /** Number of replications per judgment */
        replications: number;
        /** Total judgments analyzed */
        totalJudgments: number;
        /** Recommended noise threshold for comparisons (2× max dimension σ) */
        recommendedThreshold: number;
        /** Per-dimension consistency (keyed by dimension name) */
        perDimension: Record<string, {
            avgStdDev: number;
            maxStdDev: number;
        }>;
    };
    /** Grader model used for this evaluation */
    graderModel: string;
    /** Sensitivity / discrimination power (from degradation testing) — Phase 4 */
    sensitivity?: {
        /** % of paired comparisons where grader ranked original higher */
        concordanceRate: number;
        /** Average score separation (original - degraded) */
        avgSeparation: number;
        /** Total paired comparisons analyzed */
        totalPairs: number;
        /** Per-dimension sensitivity (keyed by dimension name) */
        perDimension: Record<string, {
            concordanceRate: number;
            avgSeparation: number;
        }>;
    };
    /** Criterion validity (from human reference grades) — Phase 2 */
    validity?: {
        /** Mean Absolute Error vs. human grades */
        mae: number;
        /** Pearson correlation with human grades */
        correlation: number;
        /** Systematic bias (positive = grader scores higher than humans) */
        bias: number;
        /** Per-dimension correlation with human grades (keyed by dimension name) */
        perDimension: Record<string, number>;
        /** Number of human-graded reference samples */
        sampleSize: number;
        /** Whether the grader passes the MAE threshold */
        passesThreshold: boolean;
    };
}
/** Cost estimate for targeted LOO before execution */
export interface LOOCostEstimate {
    /** Breakdown per ambiguous task */
    perTask: {
        /** Estimated cost for this task's LOO */
        estimatedCost: number;
        /** Number of documents to evaluate */
        numDocuments: number;
        /** Number of tests per document */
        numTests: number;
        /** Task ID */
        taskId: string;
    }[];
    /** Total estimated additional cost */
    totalEstimatedCost: number;
}
/** Result of targeted leave-one-out attribution for an ambiguous task */
export interface LOOResult {
    /** Additional eval cost incurred for this LOO analysis */
    additionalCost: number;
    /** Per-document marginal contributions */
    contributions: {
        /** Marginal score contribution of this document */
        marginalContribution: number;
        /** The document slug */
        slug: string;
        /** Whether the contribution is within grader noise floor */
        withinNoiseFloor: boolean;
    }[];
    /** Whether contributions sum to approximately the total delta (±15%) */
    sumMatchesTotal: boolean;
    /** The task being analyzed */
    taskId: string;
}
/** A parsed model entry from config/models.yaml */
export interface ModelEntry {
    config?: Record<string, unknown>;
    /** Environment variable gate — model is skipped when this var is unset */
    env?: string;
    id: string;
    label: string;
    /**
     * Which evaluation modes this model participates in.
     *
     * Values must be canonical eval mode names (e.g., "literacy", "mcp-server").
     * When omitted, the model participates in all modes.
     */
    modes?: EvalMode[];
    /**
     * Per-provider timeout in ms. Emitted into Promptfoo provider config.
     * Default: 300_000 (5 min, matching Promptfoo's built-in default).
     */
    timeoutMs?: number;
    /**
     * Per-mode variant whitelist. Keys are eval mode IDs, values are arrays
     * of variant IDs to include for that mode.
     *
     * When a model enrolls in a mode (via `modes`) but does not specify
     * variants for it here, ALL variants defined by the mode base are included.
     *
     * Only meaningful for modes that define variants (e.g., literacy has
     * "baseline", "observed", "agentic-naive", "agentic-optimized").
     * Ignored for modes without variants.
     */
    variants?: Partial<Record<EvalMode, string[]>>;
}
/** Parsed config/models.yaml structure */
export interface ModelsConfig {
    defaults: Record<string, unknown>;
    /**
     * Maximum wall-clock time per eval step (all tests for one mode) in ms.
     * When exceeded, the subprocess is killed and partial results are used.
     * Default: no limit (backward compatible).
     */
    evalBudgetMs?: number;
    grader: {
        id: string;
        label?: string;
    };
    /** Global max concurrency across all providers */
    maxConcurrency?: number;
    models: ModelEntry[];
}
/** Per-model score entry — one element per evaluated model */
export interface PerModelEntry {
    /** Model label (human-readable name) */
    label: string;
    /** Provider model ID (e.g., "openai:chat:gpt-4o") */
    modelId: string;
    /** Model-level aggregates */
    overall: {
        avgDocLift: number;
        avgScore: number;
        /** Total cost for this model's tests */
        cost?: number;
        /** Number of tests run for this model */
        testCount: number;
    };
    /** Per-area scores for this model */
    scores: FeatureScore[];
}
/** Per-model score breakdown within a single evaluation run */
export type PerModelScores = PerModelEntry[];
/** Options for a pipeline run */
export interface PipelineOptions {
    /** Debug mode — run a subset of tests for speed */
    debug?: DebugOptions;
    /** Run validation only, don't execute steps */
    dryRun: boolean;
    /** Scope evaluation to specific areas or tasks */
    filter?: FilterOptions;
    /** Which evaluation mode to run */
    mode: EvalMode;
    /** Bypass all pipeline caching (force re-execution of every step) */
    noCache: boolean;
    /** Skip the eval step (recalculate from existing results) */
    skipEval: boolean;
    /** Skip the documentation fetch step (reuse cached contexts) */
    skipFetch: boolean;
    /** Documentation source name (from config/sources.yaml) */
    source?: string;
}
/** A Promptfoo share URL tagged with the evaluation mode that produced it. */
export interface PromptfooUrlEntry {
    mode: string;
    url: string;
}
/**
 * Mutable state bag shared between pipeline steps during execution.
 *
 * Flows through the orchestrator loop — producer steps write to it,
 * consumer steps read from it. This eliminates filesystem round-trips
 * for small inter-step coordination values (e.g., a reportId generated
 * by PublishReportStep and consumed by CallbackStep).
 *
 * Large artifacts (score-summary.json, comparison-report.json) still
 * live on the filesystem — PipelineState is for ephemeral values that
 * only matter during a single pipeline execution.
 */
export interface PipelineState {
    /**
     * Artifact refs produced by upstream steps during the run.
     * Populated incrementally (CalculateScoresStep writes testOutputs; future
     * steps will write renderedPrompts, traces, etc.). Read by FinalizeRunStep
     * to build the `RunManifest.artifacts` catalog.
     */
    artifactRefs?: Partial<ArtifactManifest>;
    /**
     * The run manifest, finalized and written to `runs/{runId}/manifest.json`.
     * Populated by FinalizeRunStep. Consumed by PublishReportStep to snapshot
     * the `artifacts` slice into `Report.artifactManifest` (D0032).
     */
    runManifest?: RunManifest;
    /** Report ID generated by PublishReportStep, consumed by CallbackStep + orchestrator job update */
    reportId?: string;
    /** The stamped report, set by PublishReportStep for downstream steps. */
    report?: Report;
    /** Eval fingerprint computed by RunEvalStep, consumed by PublishReportStep */
    evalFingerprint?: string;
    /**
     * Every eval sub-run's fingerprint, accumulated across RunEvalStep
     * executions (a FULL literacy run has two: baseline + agentic). Produced by
     * RunEvalStep, consumed by PublishReportStep, which persists all of them on
     * `provenance.evalFingerprints` so an unchanged re-run cache-hits on every
     * sub-run. `evalFingerprint` above holds only the last one written.
     */
    evalFingerprints?: string[];
    /** Promptfoo share URLs produced by RunEvalStep, consumed by PublishReportStep */
    promptfooUrls?: PromptfooUrlEntry[];
    /**
     * Eval modes that were satisfied by a remote cache hit (score-summary.json
     * was restored from the Content Lake). Produced by RunEvalStep, consumed by
     * CalculateScoresStep to skip re-calculation when all required modes are cached.
     */
    remoteCacheHits?: Set<string>;
    /**
     * Release auto-scope metadata. Set by FetchDocsStep when a perspective
     * is active and release impact identifies affected documents.
     * Consumed by GenerateConfigsStep and RunEvalStep to narrow scope.
     */
    releaseAutoScope?: ReleaseAutoScope;
    /**
     * Literacy task IDs that FetchDocsStep dropped because their declared
     * canonical doc refs resolved to empty/missing context (renamed, removed,
     * or typo'd docs, or refs pointing outside the corpus). Recorded so the
     * downstream steps that independently re-load tasks — GenerateConfigsStep
     * and RunEvalStep — exclude them too, instead of one unresolvable ref
     * aborting the whole multi-task run. The skip is always logged, never
     * silent (keeps faith with D0066: a narrowed corpus stays detectable).
     */
    contextEmptyTaskIds?: string[];
    /**
     * Impact of the active source-level release perspective on the doc
     * corpus, including the `uncovered` visibility net. Set by FetchDocsStep
     * whenever a perspective diff ran — independently of `releaseAutoScope`,
     * which stays unset when every task is affected (auto-scope adds no
     * value) or auto-scoping is disabled. Consumed by PublishReportStep so
     * the published report's `provenance.autoScope` carries the release's
     * uncovered edits even when no scope narrowing occurred.
     */
    releaseImpact?: {
        perspective: string;
        impact: ReleaseImpact;
    };
    /**
     * Feature areas that scored below the critical threshold (40).
     * Set by CalculateScoresStep, consumed by the orchestrator for
     * final pipeline result reporting. The pipeline continues running
     * (gap-analysis, publish, report, compare) even when areas are
     * below threshold — this is informational, not a hard failure.
     */
    belowCritical?: string[];
    /**
     * Test execution summary. Set by CalculateScoresStep which reads
     * the eval results and counts passed/failed/errored tests.
     * Consumed by the orchestrator for the enriched PipelineResult.
     */
    testSummary?: TestSummary;
    /**
     * W0198 deterministic-lane reports keyed by `${run}/${mode}/${task}/${model}`.
     * Set by RunEvalStep after `emitSymbolPreflight`, consumed by the
     * scoring step (Phase 5) so the per-test merge does not need to read
     * the artifact back from disk. The map is intentionally small — one
     * entry per (task, model) per mode.
     */
    preflightReports?: Map<string, SymbolPreflightReport>;
}
/**
 * Release auto-scope metadata — which tasks are affected by a content
 * release's document changes. Produced by FetchDocsStep, consumed by
 * GenerateConfigsStep and RunEvalStep to narrow the evaluation scope.
 */
export interface ReleaseAutoScope {
    /** Task IDs whose canonical docs are affected by the release */
    affectedTaskIds: string[];
    /** Task IDs that were skipped (no affected docs) */
    skippedTaskIds: string[];
    /** The perspective ID that triggered scoping */
    perspective: string;
    /** The release impact data (doc slugs by change type) */
    impact: {
        added: string[];
        modified: string[];
        removed: string[];
        unchanged: string[];
    };
}
/** Result of a full pipeline run */
/** Classified failure reason for a pipeline run. */
export type PipelineFailureReason = {
    type: "validation";
    message: string;
} | {
    type: "missing-results";
    message: string;
} | {
    type: "step-failed";
    step: string;
    message: string;
} | {
    type: "all-tests-errored";
    message: string;
};
/** Summary of test execution outcomes. */
export interface TestSummary {
    /** Total test cases executed */
    total: number;
    /** Tests that passed grading */
    passed: number;
    /** Tests that failed grading */
    failed: number;
    /** Tests that errored (API timeout, malformed response, etc.) */
    errored: number;
    /** Details of errored tests (model, task, error message) */
    errors?: {
        model: string;
        task: string;
        error: string;
    }[];
    /** Per-test timing statistics (when latencyMs is available from Promptfoo) */
    timing?: {
        /** Median test duration in ms */
        medianMs: number;
        /** 95th percentile test duration in ms */
        p95Ms: number;
        /** Maximum test duration in ms */
        maxMs: number;
        /** Tests that exceeded the slow threshold (2x median, min 60s) */
        slowTests: {
            task: string;
            model: string;
            durationMs: number;
        }[];
    };
}
/** Token usage and estimated cost for a pipeline run. */
export interface PipelineUsage {
    totalTokens: number;
    evalTokens: number;
    graderTokens: number;
    estimatedCostUsd?: number;
}
export interface PipelineResult {
    /** Feature areas that scored below the critical threshold. Informational — not a failure. */
    belowCritical?: string[];
    /** Cache hit/miss statistics for this run */
    cache?: {
        hits: number;
        misses: number;
        skipped: number;
        total: number;
    };
    /** Total duration in milliseconds */
    durationMs: number;
    /** Classified failure reason (when success is false). */
    failureReason?: PipelineFailureReason;
    /** @deprecated Use `promptfooUrls` — kept for backward compatibility */
    promptfooUrl?: string;
    /** Per-mode Promptfoo share URLs (one per sub-eval that produced a shareable link) */
    promptfooUrls?: PromptfooUrlEntry[];
    /** Results per step */
    steps: Record<string, StepResult>;
    /** Report ID produced by PublishReportStep (when publish was enabled). Used by
     * post-run hooks (e.g. runPostPipelineHooks) to target diagnosis and telemetry
     * writeback at the correct Content Lake document. Absent when publish was
     * skipped or the publish step did not produce a report. (Phase 6 / DIAG-06)
     */
    reportId?: string;
    /** Overall success (all non-skipped steps succeeded) */
    success: boolean;
    /** Summary of test execution outcomes. */
    testSummary?: TestSummary;
    /** Token usage and estimated cost. */
    usage?: PipelineUsage;
    /** Validation issues found (if any) */
    validation: ValidationResult;
}
/** Result of a Tier B probe (new feature, no tasks) */
export interface ProbeResult {
    /** API names the model extracted from the docs */
    apiNamesExtracted: string[];
    /** The documents evaluated */
    documentSlugs: string[];
    /** API names the model used that are NOT in the docs (hallucinations) */
    hallucinatedApis: string[];
    /** Whether the model produced any code output */
    producedCode: boolean;
    /** The model's raw output (for human review) */
    rawOutput: string;
    /** Suggested task descriptions based on document content analysis */
    suggestedTasks: string[];
    /** Qualitative assessment */
    usability: "not-usable" | "partially-usable" | "usable";
}
/** A product feature in the registry */
export interface ProductFeature {
    /** Maps to tasks/<area>.yaml */
    area?: string;
    /** Unique feature identifier */
    id: string;
    /** Human-readable name */
    name: string;
    /** Priority for evaluation coverage */
    priority: "critical" | "high" | "low" | "medium";
    /** Product sections this feature belongs to */
    sections: string[];
    /** Coverage status */
    status: "covered" | "out-of-scope" | "planned" | "uncovered";
}
/** Full classification of a content release for evaluation */
export interface ReleaseClassification {
    /** All classified documents in the release */
    documents: ClassifiedReleaseDocument[];
    /** Timestamp of classification */
    generatedAt: string;
    /** Summary counts */
    summary: {
        /** New documents in areas with existing tasks */
        tierA: number;
        /** New documents in areas without tasks */
        tierB: number;
        /** Total documents in the release */
        total: number;
        /** Documents tracked by at least one task */
        tracked: number;
        /** Updated documents (existing in canonical sets) */
        updated: number;
        /** Documents not tracked by any task */
        untracked: number;
    };
}
/** Cross-area release impact report */
export interface ReleaseImpactReport {
    /** Per-area impact summaries */
    areas: {
        /** Area name */
        area: string;
        /** Overall area score delta */
        delta: number;
        /** Whether the area regressed (delta < -threshold) */
        regressed: boolean;
        /** Per-task deltas within this area */
        tasks: {
            /** Documents attributed to this delta */
            attributedDocs: string[];
            /** Task score delta */
            delta: number;
            /** Task ID */
            taskId: string;
        }[];
    }[];
    /** Areas evaluated but with no score change (within noise) */
    confirmedUnchanged: string[];
    /** Classified documents from the release */
    documents: ClassifiedReleaseDocument[];
    /** Timestamp of report */
    generatedAt: string;
    /** Areas not evaluated (no release documents match) */
    notEvaluated: string[];
    /** Overall score delta across all affected areas */
    overallDelta: number;
    /** Probe results for Tier B documents (if any) */
    probes: ProbeResult[];
    /** Areas that regressed (delta < -threshold) */
    regressions: string[];
}
/** Top-level retrieval metrics (added to ScoreSummary when agentic data exists) */
export interface RetrievalMetrics {
    /** Per-area retrieval metrics */
    areas: AreaRetrievalMetrics[];
    /** All canonical doc slugs that were never retrieved by any task */
    globalInvisibleDocs: string[];
    /** Overall averages across all areas */
    overall: {
        avgF1: number;
        avgPrecision: number;
        avgRecall: number;
    };
}
/** Per-feature agent behavior data — how agents interacted with docs */
export interface FeatureAgentBehavior {
    avgDocPagesVisited: number;
    avgNetworkTimeMs: number;
    avgSearchesPerformed: number;
    docSlugsVisited: string[];
    externalDomains: string[];
    feature: string;
    searchQueries: string[];
    tasksWithBehaviorData: number;
}
/** Aggregate agent behavior stats across all features */
export interface OverallAgentBehavior {
    avgDocPagesVisited: number;
    avgNetworkTimeMs: number;
    avgSearchesPerformed: number;
    testsWithBehaviorData: number;
    totalUniqueDocSlugs: number;
    totalUniqueSearchQueries: number;
}
/** Top-level score summary (the shape of score-summary.json) */
export interface ScoreSummary {
    /** Per-feature agent behavior data (only present when agentic mode ran) */
    agentBehavior?: FeatureAgentBehavior[];
    belowCritical: string[];
    /**
     * All Sanity documents used across the entire evaluation.
     * De-duplicated union of documents from all areas/tasks.
     * Populated during the gap-analysis enrichment step from the document manifest.
     */
    documentManifest?: DocumentRef[];
    /**
     * Which evaluation variant contributed data to this summary.
     *
     * For literacy mode this is a variant name:
     * - `'full'`: both baseline and agentic data present
     * - `'baseline'`: floor + ceiling only (no agentic data)
     * - `'agentic'`: actual only (no floor/ceiling data)
     * - `'observed'`: observed mode data
     *
     * For non-literacy modes this is the canonical mode name.
     * Absent in legacy summaries — treat as `'baseline'` for backward compat.
     */
    evaluationMode?: string;
    /** Failure mode analysis (Phase 3a) — diagnostic breakdown of why scores are low */
    failureModes?: FailureModeReport;
    /**
     * Typed error signal (eval failure observability, B1). Distinct from
     * `failureModes` (graded doc-quality judgments) — `errorSummary` covers
     * tests that produced NO gradeable output at all. Optional + additive;
     * rides along unchanged on cache-served runs (the slim `ReportSummary`
     * spreads it through untouched). See `ErrorSummary` in storage-schema.ts.
     */
    errorSummary?: ErrorSummary;
    /**
     * Low-scoring grader judgments (normalized score < 70) — the raw "red text"
     * from the grader explaining why specific tests scored poorly.
     * Scores are normalized to 0–100 scale before filtering.
     * Each judgment includes the canonical docs the task expected.
     */
    lowScoringJudgments?: StoredJudgment[];
    /** Gap analysis recommendations (Phase 3b) — prioritized remediation plan */
    recommendations?: GapAnalysisReport;
    /**
     * Per-test results with model output, grader reasoning, and metadata.
     * One entry per test × model combination. Populated during gap-analysis
     * enrichment from test-results.json. See D0029.
     */
    testResults?: StoredTestResult[];
    /** Grader reliability diagnostics (does not affect scores) */
    graderReliability?: GraderReliability;
    lowestArea: string;
    lowestScore: number;
    /** Areas with negative Doc Lift — docs hurt performance. Empty when all lifts are positive. */
    negativeDocLiftAreas?: {
        area: string;
        docLift: number;
    }[];
    overall: {
        /** Aggregate agent behavior stats (only present when agentic mode ran) */
        agentBehavior?: OverallAgentBehavior;
        /** Average actual (agentic) score across areas. Absent if no agentic data. */
        avgActualScore?: number;
        /** Average ceiling score across all areas */
        avgCeilingScore: number;
        avgScore: number;
        avgDocLift: number;
        /** Average doc quality gap across all areas (100 − ceiling) */
        avgDocQualityGap: number;
        /** Average floor score across all areas */
        avgFloorScore: number;
        /** Average infrastructure efficiency (actual / ceiling). Absent if no agentic data. */
        avgInfrastructureEfficiency?: number;
        /** Average retrieval gap (ceiling − actual). Absent if no agentic data. */
        avgRetrievalGap?: number;
        /** Count of areas with negative Doc Lift */
        negativeDocLiftCount: number;
        cost?: {
            total: number;
            perTest: number;
            graderTotal: number;
            graderModel?: string;
            totalTokens: number;
        };
    };
    /** Per-model score breakdowns (Tier 3 from metrics-design.md) */
    perModel?: PerModelScores;
    /** Retrieval metrics from agentic mode (Approach 4) */
    retrievalMetrics?: RetrievalMetrics;
    scores: FeatureScore[];
    source?: {
        name: string;
        baseUrl: string;
        dataset?: string;
        /** Sanity release perspective ID (when evaluating content releases) */
        perspective?: string;
        projectId: string;
    };
    timestamp: string;
}
/**
 * The subset of `ScoreSummary` the `compare()` primitive reads and the
 * only fields consumers of `ComparisonReport.baseline` / `.experiment`
 * access at runtime. Both the full pipeline `ScoreSummary` and the slim
 * `ReportSummary` (W0051) carry these fields unchanged, so stored Reports
 * can participate in auto-compare without re-hydrating prose fields.
 */
export type ComparableSummary = Pick<ScoreSummary, "overall" | "perModel" | "scores">;
/**
 * Slim pointer to a single low-scoring grader judgment. Replaces the full
 * `StoredJudgment` inlined on pre-W0051 reports. The `id` field IS the
 * `graderJudgments` manifest entry key — Studio looks up `reasonPreview`
 * there for list rendering and hydrates the full reason on drill-down.
 *
 * `reasonPreview` is ALSO carried inline as a graceful-degradation fallback
 * for runs whose `artifactManifest` is empty (offline dev, failed GCS
 * upload, cache-skip before manifest aggregation lands). At ~280 chars per
 * judgment × the 50-judgment cap ≈ 14 KB on the Report — tiny against the
 * 500 KB budget. Studio's dispatch prefers the manifest-entry preview when
 * present so GCS drill-down still renders the hydrated-on-demand copy.
 *
 * `graderId` replaces the historical `dimension` field at the slim-shape
 * boundary to match D0033 axis naming; runtime value is identical.
 */
export interface SlimJudgmentRef {
    /** Manifest entry key = `formatEntryKey({mode, task, model, grader})` for `graderJudgments`. */
    id: string;
    taskId: string;
    modelId: string;
    /** Rubric dimension name at runtime (what `GraderJudgment.dimension` carries). */
    graderId: string;
    /**
     * Alias of `graderId` — carried for pre-W0051 Studio readers that still
     * access `.dimension` (JudgmentList, judgment-formatting, et al). Remove
     * in Slice 6 when those consumers migrate to `graderId`. Identical
     * runtime value; no additional ambiguity.
     */
    dimension: string;
    /** Normalized 0–100 score. */
    score: number;
    /**
     * Truncated grader reason (≤280 chars). Inline fallback used when the
     * manifest entry's preview is unavailable; the authoritative full reason
     * lives in the `graderJudgments` external artifact.
     */
    reasonPreview?: string;
    /**
     * Alias of `reasonPreview` — legacy Studio renderers read `.reason`;
     * expose the same truncated text under both names for the compat window.
     * Remove alongside `dimension` in Slice 6.
     */
    reason?: string;
}
/**
 * Slim failure-mode entry on the Report summary. One per classified
 * `FailureModeType` bucket; `id` is the `failureModes` manifest entry key
 * so drill-down can fetch the full category payload.
 */
export interface SlimFailureModeTopTitle {
    /** Manifest entry key = `formatEntryKey({mode, category})` for `failureModes`. */
    id: string;
    category: string;
    severity: "low" | "medium" | "high" | "critical";
    title: string;
    count: number;
}
/** Counts + top-N per-category summary on the Report. */
export interface SlimFailureModesSummary {
    /** Count by FailureModeType-ish id. */
    counts: Record<string, number>;
    /** Top-N categories by count, descending. */
    topTitles: SlimFailureModeTopTitle[];
    /** Total classified judgments across all categories. */
    totalJudgments: number;
    /** Percentage of judgments that landed in a non-unclassified bucket. */
    classificationRate: number;
}
/**
 * Slim gap pointer. The `gapReport` artifact is bulk (axes: `{run}`) so
 * there is no per-gap manifest entry to point at — `id` is a stable
 * synthetic composite so the UI can deduplicate and deep-link. Drill-down
 * reads the full `gapReport` artifact and filters by id.
 */
export interface SlimRecommendationGap {
    /** Synthetic id: `${area}--${failureMode}`, kebab-safe. */
    id: string;
    area: string;
    title: string;
    /** Priority bucketing for triage UI ordering. */
    priority: number;
}
/** Counts + top-3 summary on the Report. */
export interface SlimRecommendations {
    /** Count of gaps by area. */
    counts: Record<string, number>;
    /** Top-3 gaps by priority, descending. */
    top3: SlimRecommendationGap[];
    /** Total actionable gaps identified (sum of counts). */
    totalGaps: number;
    /** Aggregate estimated lift (matches `GapAnalysisReport.totalPotentialLift`). */
    totalPotentialLift: number;
}
/**
 * Slim per-feature agent-behavior summary. Full `searchQueries` and
 * `docSlugsVisited` arrays move to `traces` NDJSON; the Report keeps only
 * counts + first-N samples for triage preview. The `firstN` cap is the
 * producer's choice (we default to 5).
 */
export interface SlimAgentBehaviorFeature {
    feature: string;
    avgDocPagesVisited: number;
    avgNetworkTimeMs: number;
    avgSearchesPerformed: number;
    tasksWithBehaviorData: number;
    externalDomains: string[];
    /** Distinct count of unique search queries across tasks in this feature. */
    searchQueriesCount: number;
    /** First-N unique search queries (bounded samples). */
    searchQueriesSample: string[];
    /** Distinct count of unique doc slugs visited. */
    docSlugsVisitedCount: number;
    /** First-N unique doc slugs (bounded samples). */
    docSlugsVisitedSample: string[];
}
/**
 * Slim `summary` field on a published `Report`. Structurally
 * `Omit<ScoreSummary, slimmed-fields> & slim-replacements` — every
 * pipeline-produced field survives except the four prose/array fields
 * W0051 moves to external artifacts.
 */
export type ReportSummary = Omit<ScoreSummary, "agentBehavior" | "failureModes" | "lowScoringJudgments" | "recommendations"> & {
    agentBehavior?: SlimAgentBehaviorFeature[];
    failureModes?: SlimFailureModesSummary;
    lowScoringJudgments?: SlimJudgmentRef[];
    recommendations?: SlimRecommendations;
};
/** Result of a single pipeline step */
export type StepResult = {
    status: "failed";
    durationMs: number;
    error: string;
} | {
    status: "skipped";
    reason: string;
} | {
    status: "success";
    durationMs: number;
    summary: string;
};
/** Per-task attribution result — which documents caused a score change */
export interface TaskAttribution {
    /** The feature area */
    area: string;
    /** Documents attributed (1 for unambiguous, N for ambiguous, 0 for uncorrelated) */
    attributedDocs: string[];
    /** Attribution classification */
    classification: AttributionClass;
    /** Score delta for this task (area-level delta used as proxy) */
    delta: number;
    /** The task ID */
    taskId: string;
    /** Whether delta is within grader noise floor */
    withinNoiseFloor: boolean;
}
/** A parsed task entry from a tasks/*.yaml file (single-definition format) */
export interface TaskEntry {
    assert?: unknown[];
    /** Baseline generation options (new format only). */
    baseline?: {
        enabled?: boolean;
        rubric?: "abbreviated" | "full" | "none";
    };
    description: string;
    /** Explicit task ID — determines the canonical context filename. */
    id?: string;
    transform?: string;
    vars?: Record<string, unknown>;
}
/** Retrieval metrics for a single task */
export interface TaskRetrievalMetrics {
    /** Feature area (from task file) */
    area: string;
    /** Slugs expected (from canonical_docs) */
    expected: string[];
    /** F1 score: 2 * precision * recall / (precision + recall) */
    f1: number;
    /** Slugs in both retrieved and expected */
    hits: string[];
    /** Expected slugs the agent missed */
    missed: string[];
    /** Precision: |retrieved ∩ canonical| / |retrieved| */
    precision: number;
    /** Recall: |retrieved ∩ canonical| / |canonical| */
    recall: number;
    /** Slugs the agent actually retrieved */
    retrieved: string[];
    /** Task ID matching the task YAML definition */
    taskId: string;
    /** Retrieved slugs not in the expected set */
    unexpected: string[];
}
/** A single validation finding — either an error (blocks execution) or a warning */
export interface ValidationIssue {
    /** What went wrong */
    message: string;
    /** Optional file path relevant to the issue */
    path?: string;
    /** Error = blocks execution, warning = informational */
    severity: ValidationSeverity;
    /** Which step or check produced this finding */
    source: string;
}
/** Result of a validation pass */
export interface ValidationResult {
    /** All findings */
    issues: ValidationIssue[];
    /** True if there are no errors (warnings are OK) */
    valid: boolean;
}
/** Severity of a validation finding */
export type ValidationSeverity = "error" | "warning";
/**
 * Default noise threshold — re-exported from @sanity/ailf-shared.
 * Named DEFAULT_NOISE_THRESHOLD here for backward compatibility with eval code
 * that imports it from @sanity/ailf-core.
 */
export { NOISE_THRESHOLD as DEFAULT_NOISE_THRESHOLD } from "../../ailf-shared/index.d.ts";
/** Per-area delta with dimension breakdowns */
export interface AreaDelta {
    /** Actual (agentic) score delta. Absent if either run lacks agentic data. */
    actualDelta?: number;
    area: string;
    /** Baseline total score */
    baseline: number;
    /** Ceiling score delta (doc quality change). If ceiling moved, doc content changed. */
    ceilingDelta: number;
    /** Change classification based on noise threshold */
    change: ChangeClass;
    /** Cost delta (if cost data is available) */
    costDelta?: number;
    /**
     * Overall score delta (experiment − baseline). `undefined` when the area is
     * not comparable — it was scored in only one of the two runs
     * (`change === "not-evaluated"`), so there is no honest delta. Consumers must
     * treat `undefined` as "no comparable baseline", never as 0 (W-mark-uncomparable-areas).
     */
    delta?: number;
    /** Per-dimension deltas (keyed by dimension name, e.g. 'taskCompletion') */
    dimensions: Record<string, {
        baseline: number;
        experiment: number;
        delta: number;
    }>;
    /** Doc Lift delta */
    docLiftDelta: number;
    /** Experiment total score */
    experiment: number;
    /** Floor score delta (model knowledge change — should be ~0 between runs) */
    floorDelta: number;
    /** Infrastructure efficiency delta. Absent if either run lacks full decomposition data. */
    infrastructureEfficiencyDelta?: number;
    /** Retrieval gap delta. Positive = gap grew (worse), negative = gap shrank (better). */
    retrievalGapDelta?: number;
}
/** Attribution classification for a task delta */
export type AttributionClass = "ambiguous" | "unambiguous" | "uncorrelated";
/** Complete attribution report for a set of changed documents */
export interface AttributionReport {
    /** All task attributions */
    attributions: TaskAttribution[];
    /** Summary counts */
    summary: {
        ambiguous: number;
        unambiguous: number;
        uncorrelated: number;
        withinNoise: number;
    };
    /** Changed documents that aren't in any task's canonical docs */
    untrackedDocs: string[];
}
/** Classification of a feature area's score change */
export type ChangeClass = "improved" | "not-evaluated" | "regressed" | "unchanged";
/** Options for the compare function */
export interface CompareOptions {
    /** Grader consistency data — if provided, used to compute empirical noise threshold */
    graderConsistency?: GraderConsistencyData;
    /**
     * Deltas within ±threshold are classified as "unchanged".
     * Defaults to DEFAULT_NOISE_THRESHOLD (2).
     * Overridden by graderConsistency.recommendedThreshold when available.
     */
    noiseThreshold?: number;
}
/**
 * Structured comparison between two evaluation runs.
 *
 * This is the core comparison primitive (see BP5 in the evaluation roadmap).
 * Every comparison scenario uses the same shape — what varies is what
 * produced each ScoreSummary:
 *
 * | Scenario            | Baseline            | Experiment         |
 * |---------------------|---------------------|--------------------|
 * | Doc improvement     | Before rewrite      | After rewrite      |
 * | Model comparison    | Model A             | Model B            |
 * | Model regression    | Previous version    | Current version    |
 * | Branch validation   | Production          | Branch deploy      |
 * | Infrastructure ROI  | Naive agent         | Optimized agent    |
 * | Training freshness  | Without docs        | With docs (DocLift)|
 */
export interface ComparisonReport {
    /** Detailed per-area breakdowns */
    areas: AreaDelta[];
    /** Per-document attribution (when changed docs are known) */
    attribution?: AttributionReport;
    /** The "before" or "control" summary (narrowed in W0051 so slim Reports compare) */
    baseline: ComparableSummary;
    /**
     * Per-area baseline attribution (W-resolve-per-area-baselines). Present only
     * when the comparison resolved each area's baseline independently (the
     * scheduled/fuzzy auto-compare path), recording which prior report supplied
     * each area's baseline score. Absent for single-baseline comparisons (a
     * pinned `compareBaselineReportId`, a `rerunOf` lineage, or legacy reports),
     * where the whole experiment diffs against one baseline named by
     * `provenance.lineage.comparedAgainst`.
     *
     * Array shape (not a keyed map) per D0041 — each new feature area would
     * otherwise mint a permanent Sanity attribute path. In the common homogeneous
     * case every entry names the same report; the array still records it per-area
     * so a heterogeneous run (areas diffed against different prior dates) is
     * legible.
     */
    baselineSources?: {
        area: string;
        completedAt: string;
        reportId: string;
    }[];
    /** Aggregate deltas */
    deltas: {
        /** Overall score delta (experiment.avgScore − baseline.avgScore) */
        overall: number;
        /**
         * Per-area total score deltas (array shape — W0137 / D0041). Only areas
         * scored in BOTH runs appear here; areas scored in one run
         * (`change === "not-evaluated"`) are omitted so a missing entry reads as
         * "no comparable baseline" rather than a fabricated `score − 0` delta
         * (W-mark-uncomparable-areas).
         */
        perArea: {
            area: string;
            delta: number;
        }[];
        /** Per-dimension average deltas (across all areas) */
        perDimension: Record<string, number>;
        /** Doc Lift average delta */
        docLift: number;
        /** Cost delta (if both runs have cost data) */
        cost?: number;
        /** Per-model overall score deltas (when both summaries have per-model data) */
        perModel?: {
            delta: number;
            modelId: string;
        }[];
    };
    /** The "after" or "treatment" summary (narrowed in W0051 so slim Reports compare) */
    experiment: ComparableSummary;
    /** When this comparison was generated */
    generatedAt: string;
    /** Areas that improved beyond the noise threshold */
    improved: string[];
    /** Areas present in only one of the two summaries */
    mismatched: {
        /** Areas in baseline but not experiment */
        onlyInBaseline: string[];
        /** Areas in experiment but not baseline */
        onlyInExperiment: string[];
    };
    /** Noise threshold used for classification */
    noiseThreshold: number;
    /** Whether the noise threshold was derived from empirical grader consistency data */
    noiseThresholdEmpirical: boolean;
    /**
     * Areas present in only one summary (not evaluated in both runs).
     * These are excluded from improved/regressed/unchanged classification
     * because comparing against a missing score is meaningless.
     */
    notEvaluated: string[];
    /** Areas that regressed beyond the noise threshold */
    regressed: string[];
    /** Areas within the noise threshold */
    unchanged: string[];
}
/** Confidence annotation for a single delta */
export interface ConfidenceAnnotation {
    /** 95% confidence interval half-width (±this value) */
    ci95: number;
    /** The measured delta */
    delta: number;
    /** The grader σ used to compute this CI */
    graderSigma: number;
    /** Whether the delta exceeds the confidence interval (statistically significant) */
    significant: boolean;
}
/**
 * Grader consistency data for empirical confidence intervals.
 * This is the subset of GraderConsistency needed for comparison annotation.
 * Matches the shape produced by pipeline/grader-consistency.ts.
 */
export interface GraderConsistencyData {
    /** Per-dimension consistency metrics (keyed by dimension name) */
    perDimension: Record<string, {
        avgStdDev: number;
        maxStdDev: number;
    }>;
    /** Recommended noise threshold for comparisons (2× max dimension σ) */
    recommendedThreshold: number;
}
/** ISO 8601 timestamp string */
export type ISOTimestamp = string & {
    readonly __brand: "ISOTimestamp";
};
/** Query parameters for finding comparable baseline reports */
export interface LineageQuery {
    /** Match on feature areas (overlapping set) */
    areas?: string[];
    /** Only reports before this timestamp */
    before?: ISOTimestamp;
    /**
     * Match on run classification. `classification` partitions runs into trend
     * series (D0037) — an `official` daily run is only comparable to other
     * `official` runs, never to a `test`/`adhoc`/`experimental` run that merely
     * shares its mode and source. Comparing across classifications mixes a
     * canonical-series average with throwaway runs, so baseline selection treats a
     * classification mismatch as uncomparable, just as it does a variant mismatch.
     */
    classification?: RunClassification;
    /** Match on evaluation mode */
    mode: EvalMode;
    /** Match on models */
    models?: string[];
    /** Match on source */
    source?: Partial<ReportProvenance["source"]>;
    /**
     * Match on literacy variant. Comparing across variants is apples-to-oranges
     * — the scoring methodology differs per variant (`baseline` vs `agentic` vs
     * `full`) — so baseline selection treats a variant mismatch as uncomparable.
     * Only meaningful when `mode === "literacy"`; absent for other modes.
     */
    variant?: LiteracyVariant;
}
/** Result of publishing a report to all sinks */
export interface PublishResult {
    /** The auto-comparison (if a comparable baseline was found) */
    comparison?: ComparisonReport;
    /** The published report */
    report: Report;
    /** Per-sink results */
    sinkResults: {
        name: string;
        result: SinkResult;
    }[];
}
/**
 * A single per-entry row in `ArtifactRef.entries`. Carries enough metadata for
 * Studio list/triage views to render without fetching the external payload.
 *
 * D0033/W0049 extensions:
 *   - `association` — the axis values that identify this entry (`{task, model}`
 *     for testOutputs, etc.). Present only on manifests written by
 *     `emit()`; legacy manifests omit it and readers treat absence as `{}`.
 *   - `truncated` — whether the entry payload was capped by the descriptor's
 *     `capBytes`. Readers treat absence as `false` (pre-W0049 manifests were
 *     never truncated because no caps were enforced).
 *   - `preview` — an inline summary produced by the descriptor's
 *     `manifestPreview.extract()`. Typed via the descriptor's preview schema;
 *     omitted when the descriptor has no `manifestPreview`. Wiring lands in
 *     W0051; the field is reserved here so manifests written now parse there.
 */
export interface ArtifactRefEntry {
    key: string;
    bytes: number;
    association?: AssociationValues;
    truncated?: boolean;
    preview?: unknown;
}
/**
 * Reference to an artifact in external object storage.
 *
 * `store` discriminates the backend: `"gcs"` uses `bucket` as the bucket
 * name (D0032); `"local"` uses `bucket` as the absolute rootDir path
 * under which `path` resolves to a file on disk (D0033 / W0050 M4).
 * Consumers (Studio retrieval, contract tests) branch on `store` only
 * when constructing the fetch URL — the `path` is store-relative and
 * identical across backends for the same logical artifact.
 *
 * `layout` determines the on-disk shape:
 *   - `"bulk"` — a single object at `path`. `entries` is absent.
 *   - `"per-entry"` — `path` is a directory prefix. Each entry is a
 *     separate object at `{path}/{sanitizedKey}.json`. `entries` inlines
 *     the catalog so consumers can render drill-down states without a
 *     second list call.
 *
 * D0033/W0049 extensions (optional — legacy manifests parse without them):
 *   - `truncated` on the bulk row indicates the single-object body was capped.
 *   - `preview` on the bulk row carries a descriptor-typed summary for list
 *     views; wiring lands in W0051.
 *
 * D0040/W0135 extension:
 *   - `sourceRunId` declares that this ref's bytes physically live under a
 *     different run's storage prefix than the manifest containing it.
 *     `path` is already self-contained and authoritative for resolution;
 *     `sourceRunId` is purely a lineage marker for retention, GC,
 *     observability, and BigQuery joins. Set by the cache-hit branch in
 *     `RunEvalStep` when a new run reuses a prior report's artifacts;
 *     unset on cold-path producers.
 */
export interface ArtifactRef {
    store: "gcs" | "local";
    /**
     * GCS bucket name for `store: "gcs"`; absolute rootDir path for
     * `store: "local"`. Kept as a single field so callers iterating
     * manifest entries don't need to branch on `store` just to read the
     * storage container.
     */
    bucket: string;
    path: string;
    bytes?: number;
    entryCount?: number;
    layout: "bulk" | "per-entry";
    entries?: ArtifactRefEntry[];
    truncated?: boolean;
    preview?: unknown;
    sourceRunId?: RunId;
}
/**
 * Catalog of artifact refs produced by a single pipeline run.
 *
 * Lives on `RunManifest.artifacts` (source of truth in GCS) and is
 * snapshotted onto `Report.artifactManifest` at publish time.
 *
 * Derived from `ArtifactType` so adding a descriptor to the registry
 * automatically admits it to the manifest catalog — drift between the
 * two becomes a compile error (W0049 review finding C1).
 */
export type ArtifactManifest = Partial<Record<ArtifactType, ArtifactRef>>;
/**
 * Enrichment surfaces that gap-analysis writes onto a report. When a full
 * eval scores tests but these are absent, the report renders as "no tests"
 * despite carrying a score — the degraded condition `ReportDegradation`
 * records.
 */
export declare const DEGRADED_ENRICHMENT_FIELDS: readonly ["documentManifest", "failureModes", "lowScoringJudgments", "recommendations", "testResults"];
export type DegradedEnrichmentField = (typeof DEGRADED_ENRICHMENT_FIELDS)[number];
/**
 * The `degraded` flag is the write-path input that stamps a report's D0059
 * `validity.status` to `"degraded"` (see {@link assessReportValidity}). It is a
 * discriminated union on `reason`; each variant carries a `missing[]` listing
 * the affected surfaces in that variant's vocabulary, plus a human-readable
 * `detail` for dashboard / Studio empty-state copy.
 *
 * Even though {@link ReportValidity} (D0059) is the go-forward data-health axis,
 * the auto detector only reaches `validity.status:"degraded"` *via* this flag —
 * so new degraded conditions are still expressed by setting `degraded`, which
 * the detector subsumes into `validity`.
 *
 * @deprecated Read shape retained for back-compat; the backfill migrates
 * `degraded → validity.status:"degraded"` (`W-backfill-report-validity`).
 */
export type ReportDegradation = ReportEnrichmentDegradation | ReportArtifactPersistenceDegradation;
/**
 * Degraded because a scored run's gap-analysis enrichment never landed: the
 * eval counted tests but `summary.testResults` (and the derived surfaces) are
 * absent, so the report renders an empty "no tests" state despite a score
 * (W0297 `detectReportDegradation`).
 */
export interface ReportEnrichmentDegradation {
    reason: "enrichment-missing";
    /** Enrichment surfaces absent on this report despite a full eval. */
    missing: DegradedEnrichmentField[];
    /** Human-readable explanation for dashboard / Studio empty-state copy. */
    detail: string;
}
/**
 * Degraded because the run's artifacts were not persisted to the remote store:
 * a remote backend (GCS / API Gateway) was configured, yet zero (or below a
 * configured threshold) of the produced artifact sections actually landed
 * there — every section fell back to `store: "local"`. The report still carries
 * valid scores and inline `testResults`, but its GCS-externalized drill-downs
 * (per-test outputs, raw results, etc.) do not exist, so a post-hoc
 * `ailf interpret` pass and the dashboard's artifact recovery have nothing to
 * read (`W-detect-wholesale-artifact-loss`).
 */
export interface ReportArtifactPersistenceDegradation {
    reason: "artifacts-not-persisted";
    /** Artifact sections that did not persist to the remote store. */
    missing: ArtifactType[];
    /** Human-readable explanation for dashboard / Studio empty-state copy. */
    detail: string;
}
/** A published evaluation report — the atomic unit of the report store */
export interface Report {
    /**
     * Set when the report is published in a degraded state — a full eval
     * scored tests but enrichment did not complete. Absent on healthy reports.
     *
     * @deprecated Superseded by {@link Report.validity} (D0059). New write paths
     * stamp `validity` (with `status:"degraded"` in this case) instead.
     */
    degraded?: ReportDegradation;
    /**
     * Post-hoc data-health assessment (D0059) — orthogonal to
     * `provenance.classification` (run intent). Top-level because it judges the
     * report's *data*, not the run. Additive + nullable: absent on pre-stamp
     * reports, which are treated as trustworthy until backfilled. Subsumes the
     * legacy {@link Report.degraded} flag.
     */
    validity?: ReportValidity;
    /**
     * Snapshot of the run manifest's `artifacts` slice at publish time (D0032).
     * The source of truth lives in `gs://…/runs/{runId}/manifest.json`; this
     * field denormalizes enough information for Studio to render drill-down
     * states without an extra manifest fetch.
     */
    artifactManifest?: ArtifactManifest;
    /** Optional auto-comparison against the most recent comparable report */
    comparison?: ComparisonReport;
    /** When the evaluation completed */
    completedAt: ISOTimestamp;
    /** How long the pipeline took */
    durationMs: number;
    /** Stable unique identifier (UUID v7 for time-sortability) */
    id: ReportId;
    /** What produced this report */
    provenance: ReportProvenance;
    /**
     * The slim published summary. Inlined prose fields (grader reasons, full
     * failure-mode text, gap prose, agent-behavior arrays) moved to external
     * artifacts in W0051; see `ReportSummary` for the retained shape and
     * `docs/decisions/D0033-unified-run-anchored-artifact-capture.md` §§ M7 for
     * the full migration table.
     */
    summary: ReportSummary;
    /** Optional human-supplied label */
    tag?: string;
    /** Auto-generated descriptive title for discoverability and sharing */
    title?: string;
}
/** Branded type for report identifiers (UUID v7 for time-sortability) */
export type ReportId = string & {
    readonly __brand: "ReportId";
};
/** Auto-scope metadata recorded in report provenance */
export interface ReportAutoScope {
    /**
     * Whether auto-scoping narrowed this evaluation to release-affected
     * tasks. `false` on a record that exists only to carry the release's
     * `uncovered` visibility net (e.g. every task was affected, so no
     * narrowing occurred) — the task-ID arrays are empty in that case.
     */
    enabled: boolean;
    /** Task IDs whose canonical docs are affected by the release */
    affectedTaskIds: string[];
    /** Task IDs that were skipped (no affected docs) */
    skippedTaskIds: string[];
    /** The perspective ID that triggered scoping */
    perspective: string;
    /** Summary of release impact (counts, not full slug lists) */
    impactSummary: {
        added: number;
        modified: number;
        removed: number;
    };
    /**
     * Release-modified doc slugs outside the evaluated canonical set
     * (`ReleaseImpact.uncovered`) — applied edits whose effect this run never
     * measured. Three-valued by presence: absent on reports published before
     * the field existed or when no release footprint was computed; an empty
     * array means "checked — every release-modified doc is covered"; a
     * non-empty array names the unmeasured docs. Count is derivable as
     * `uncovered.length`.
     */
    uncovered?: string[];
}
/**
 * Typed relationships between reports. Each field is optional and
 * independent — populated only when that relationship exists.
 *
 * Stored at `provenance.lineage` in the report document.
 *
 * @see docs/design-docs/report-store/domain-model.md
 */
export interface ReportLineage {
    /**
     * This report was explicitly compared against another report.
     * Set when auto-compare selects a specific baseline or when the user
     * requests comparison against a named report.
     */
    comparedAgainst?: ReportId;
    /**
     * This report was explicitly re-run from another report.
     * The re-run has the same EvalScope (mode, areas, perspective, etc.)
     * but measures the current state of docs/models/tasks.
     */
    rerunOf?: ReportId;
}
/**
 * Full provenance metadata for an evaluation report.
 *
 * Inherits the 9 run-description fields (mode, areas, taskIds, models,
 * graderModel, source, evalFingerprint, trigger, git) from `RunContext`.
 * Adding a field to `RunContext` makes it available here automatically —
 * the structural extension is the drift-prevention mechanism described in
 * D0032 § "Drift Prevention".
 */
export interface ReportProvenance extends RunContext {
    /** Release auto-scope metadata (when perspective evaluation was scoped to affected tasks) */
    autoScope?: ReportAutoScope;
    /** Content hash of the documentation context at eval time */
    contextHash?: string;
    /**
     * Every eval sub-run's fingerprint for this report. A FULL literacy run has
     * two sub-runs (baseline + agentic), each computing its own mode-scoped
     * fingerprint; this array carries BOTH so `ReportStore.findByFingerprint`
     * resolves the report from EITHER sub-run's fingerprint — letting an
     * unchanged full re-run cache-hit on every sub-run instead of always
     * re-running the baseline (W-persist-subrun-fingerprints). The scalar
     * `evalFingerprint` (inherited from `RunContext`) remains the last sub-run's
     * fingerprint for back-compat; reports written before this field existed
     * carry only that scalar and still resolve via the scalar match.
     */
    evalFingerprints?: string[];
    /** Typed relationships with other reports (re-run, comparison) */
    lineage?: ReportLineage;
    /** @deprecated Use `promptfooUrls` — kept for backward compatibility */
    promptfooUrl?: string;
    /** Per-mode Promptfoo share URLs (one per sub-eval that produced a shareable link) */
    promptfooUrls?: PromptfooUrlEntry[];
    /**
     * Identity of the pipeline run that produced this report. Links the
     * Content Lake document back to the GCS run manifest and its artifacts.
     * @see docs/decisions/D0032-run-anchored-artifact-store.md
     */
    runId: RunId;
    /** Sanity document IDs that were targeted (if using --sanity-document) */
    targetDocuments?: string[];
    /** Doc refs injected into task context for this run (doc-impact eval) */
    injectedContextDocs?: GeneralizedDocRef[];
}
/**
 * A run's manifest in GCS (`runs/{runId}/manifest.json`). Source of truth
 * for artifact locations, run identity, and outcome. Reports snapshot the
 * `artifacts` slice into `Report.artifactManifest` at publish time.
 *
 * Written once by `FinalizeRunStep`; `reportIds` may be appended by
 * `PublishReportStep` via strongly-consistent GCS overwrite.
 */
export interface RunManifest {
    /** Schema version — bumped when the manifest shape changes */
    version: 1;
    /** Identity for this pipeline run */
    runId: RunId;
    /** When the manifest was written (pipeline finalization time) */
    createdAt: ISOTimestamp;
    /** Total pipeline duration */
    durationMs: number;
    /** Outcome of the run */
    status: "completed" | "failed" | "partial";
    /** Failure classification when status is not "completed" */
    failureReason?: PipelineFailureReason;
    /** What ran — shared shape with ReportProvenance */
    context: RunContext;
    /** Run-level aggregates (self-describing without a report) */
    outcomes?: {
        testSummary?: TestSummary;
        usage?: PipelineUsage;
        cache?: {
            hits: number;
            misses: number;
            skipped: number;
        };
    };
    /** Reports published from this run (0..N). Authoritative link is Report.provenance.runId. */
    reportIds?: ReportId[];
    /** Promptfoo share URLs collected during the run */
    promptfooUrls?: PromptfooUrlEntry[];
    /** Artifact catalog — per-type refs with inline per-entry indexes */
    artifacts: ArtifactManifest;
}
/** Health check result for a sink */
export type SinkHealthStatus = {
    healthy: false;
    reason: string;
} | {
    healthy: true;
};
/** Payload delivered to sinks — extends Report with threshold context */
export interface SinkPayload {
    /** Comparison report (if available) */
    comparison?: ComparisonReport;
    /** Maximum severity across all violations */
    maxSeverity: "none" | ThresholdSeverity;
    /** The published report */
    report: Report;
    /** Threshold violations (empty if thresholds not configured) */
    violations: ThresholdViolation[];
}
/** Result of a sink publish attempt */
export type SinkResult = {
    status: "failed";
    error: string;
} | {
    status: "skipped";
    reason: string;
} | {
    status: "success";
    detail?: string;
};
/** Result of evaluating all thresholds against a score summary */
export interface ThresholdEvaluation {
    /** Maximum severity across all violations ("none" if no violations) */
    maxSeverity: "critical" | "info" | "none" | "warning";
    /** Whether all thresholds pass (no violations) */
    pass: boolean;
    /** All violations, sorted by severity (critical first) then by delta */
    violations: ThresholdViolation[];
}
/** Severity level for threshold violations */
export type ThresholdSeverity = "critical" | "info" | "warning";
/** A single threshold violation — one dimension or metric exceeding its limit */
export interface ThresholdViolation {
    /** The actual measured value */
    actual: number;
    /** The area (or "overall" for global thresholds) */
    area: string;
    /** Human-readable description of what violated and why */
    description: string;
    /** The dimension or metric that violated (e.g., "composite", "task-completion", "doc-lift") */
    metric: string;
    /** Severity classification based on severity config rules */
    severity: ThresholdSeverity;
    /** The threshold value that was exceeded */
    threshold: number;
}
