import { type ActualScoreEntry, type ComponentResult, type ConsumerVerificationReport, type ErrorSummary, type FeatureAgentBehavior, type GraderReliability, type Logger, type StoredTestResult, type SymbolPreflightReport, type TestResult, type TestSummary } from "../_vendor/ailf-core/index.d.ts";
import type { JudgmentConsistency } from "./grader-consistency.js";
import { type ResolvedSourceConfig } from "../sources.js";
import type { FeatureScore, GraderJudgment, PerModelEntry } from "./types.js";
import { type ScoreTestGroupOptions } from "./compiler/scoring-bridge.js";
export { classifyRubric, detectFeatureArea, extractUrlMetadata, mergeScores, parseRubricScore, type ActualScoreEntry, type ComponentResult, type StoredTestResult, type TestResult, type UrlMetadata, } from "../_vendor/ailf-core/index.d.ts";
export interface PromptfooResultsWrapper {
    results: RawTestResult[];
    stats: {
        successes: number;
        failures: number;
        tokenUsage?: {
            assertions?: {
                completion: number;
                prompt: number;
                total: number;
            };
            completion: number;
            prompt: number;
            total: number;
        };
    };
}
export interface RawPromptfooFile {
    [key: string]: unknown;
    config?: {
        defaultTest?: {
            options?: {
                provider?: string;
                rubricProvider?: string;
            };
        };
    };
    results: PromptfooResultsWrapper;
}
export interface RawTestResult {
    cost?: number;
    error?: string;
    gradingResult: null | {
        componentResults: ComponentResult[];
        pass: boolean;
    };
    /** Per-test latency in ms (populated by Promptfoo when available) */
    latencyMs?: number;
    metadata?: Record<string, unknown>;
    provider?: {
        id?: string;
        label?: string;
    };
    response: {
        output: string;
        tokenUsage?: {
            cached?: number;
            completion?: number;
            prompt?: number;
            total?: number;
        };
    };
    testCase?: {
        description?: string;
        vars?: Record<string, string>;
    };
    vars: Record<string, string>;
}
/**
 * Aggregate counts across every preflight report emitted by the run.
 *
 * `unresolvedRate` is `unresolved / totalFindings` in `[0, 1]`, set only
 * when `totalFindings > 0`. The vacuous case (`totalFindings === 0` —
 * reports exist but every candidate cited zero in-scope bindings) is
 * deliberately distinguished from "every binding resolved cleanly"
 * (`unresolvedRate === 0`) so a CI threshold like `unresolvedRate > 0.1`
 * doesn't fire green on a run that had nothing to resolve.
 */
export interface PreflightSummary {
    /** Number of per-test preflight reports the run emitted. */
    reportCount: number;
    /** Total findings across all reports. */
    totalFindings: number;
    /** Findings classified `exists`. */
    exists: number;
    /** Findings classified `missing` — the deterministic-deduction lane. */
    missing: number;
    /** Findings classified `unresolved` — the resolver-couldn't-answer lane. */
    unresolved: number;
    /**
     * `unresolved / totalFindings` in `[0, 1]`. Absent when `totalFindings`
     * is zero — distinguishes "nothing to resolve" from "all resolutions
     * succeeded" so CI thresholds aren't vacuously green.
     */
    unresolvedRate?: number;
}
/**
 * Aggregate counts across every consumer-verification report emitted by
 * the run. The block is omitted entirely when no reports exist, so its
 * presence in `score-summary.json` always means the verifier actually ran.
 */
export interface ConsumerVerificationSummary {
    /** Number of per-(task, model) verification reports the run emitted. */
    reportCount: number;
    /** Reports whose verifier verdict passed. */
    passed: number;
    /** Reports whose verifier verdict failed. */
    failed: number;
    /** Mean verifier score across reports, in `[0, 1]`. */
    meanScore: number;
}
/**
 * Calculate scores grouped by model. Each model gets its own FeatureScore[]
 * and model-level aggregates.
 *
 * Uses the provider.id from Promptfoo results to identify models.
 * Falls back to provider.label, then "unknown" if neither is available.
 *
 * @returns Record keyed by model ID, or null if only one model was used
 *          (per-model breakdown is redundant when there's only one model).
 */
export declare function calculateScoresPerModel(resultsPath: string, goldProfile: Record<string, number>, baselineProfile: Record<string, number>, scoringOptions?: ScoreTestGroupOptions): null | PerModelEntry[];
/**
 * Extract grader judgments (reason text + scores) from evaluation results.
 *
 * This preserves the grader's natural language reasoning for downstream
 * analysis (failure mode classification, gap analysis). Each llm-rubric
 * assertion produces one GraderJudgment entry.
 *
 * Phase 3a prerequisite: structured judgment data for failure mode extraction.
 *
 * @param resultsPath - Path to the Promptfoo results JSON file.
 * @param telemetry - Optional reliability counter (Plan 03-03). When passed,
 *   `parseFailures` is incremented on every strict-schema rejection so the
 *   live pipeline can surface schema drift over time.
 *   `runId` (when supplied) is threaded into synthesized fall-back judgment
 *   ids so dedup keys are unique per-run.
 */
export declare function extractGraderJudgments(resultsPath: string, telemetry?: {
    reliability: GraderReliability;
    runId?: string;
}): GraderJudgment[];
/**
 * Stamp every grader judgment with a D0049 ceiling-cross-check confidence
 * triple and increment `GraderReliability.failureModeCalibration` whenever
 * the grader's emitted `failureMode` disagrees with the
 * ceiling-decomposition mode.
 *
 * Plan 03-03 — the grader's emitted `failureMode` is the source of truth
 * for the mode itself (Plan 03-02 per-dimension taxonomies); this pass
 * stamps confidence based on whether the structural ceiling signal agrees
 * and surfaces calibration drift as a counter on `GraderReliability`.
 *
 * The function mutates `judgments` in place — it overlays
 * `judgment.confidence` with the ceiling-cross-check stamp. If a judgment
 * already carries a confidence from the strict-schema parse (Plan 03-01),
 * the ceiling-cross-check stamp REPLACES it because the validator's
 * derivation tag is the live-pipeline contract; the parsed-shape
 * confidence (if emitted by the grader) is preserved on the original
 * `parsedJudgment` upstream of this site.
 *
 * `hallucinationCheckedAgainst` is NOT populated here — `extractGraderJudgments`
 * does not have access to `task.contextDocs ∪ run.documentManifest` at this
 * site (the union travels through a separate path in
 * `gap-analysis-step.ts`'s document-enrichment flow). Plan 03-04 will
 * couple the doc-union population at the strict-schema flip site so the
 * field is populated alongside the required-flip.
 *
 * @param judgments - Grader judgments produced by `extractGraderJudgments`.
 * @param scores - Per-area feature scores; `ceilingScore` and `floorScore`
 *   come from this lookup. Missing areas default to ceiling 100, floor 0
 *   (preserves the pre-Plan-03-03 fall-back from `buildFailureModeReport`).
 * @param reliability - `GraderReliability` sink whose
 *   `failureModeCalibration` counter is incremented on disagreement.
 *
 * @see docs/decisions/D0005-grader-model-separation.md
 * @see docs/decisions/D0049-shared-confidence-contract.md
 */
export declare function validateGraderJudgmentsCalibration(judgments: GraderJudgment[], scores: FeatureScore[], reliability: GraderReliability): void;
/**
 * Populate Pitfall #11 hallucination cross-check fields on grader
 * judgments (Plan 03-04 GRAD-05).
 *
 * For each judgment, sets `hallucinationCheckedAgainst` to the union of
 * (a) the slugs of docs the task declared in `context.docs` and (b) the
 * run's full document manifest. For each entry in `judgment.docCitations`
 * that carries a `slug`, sets `hallucinated: !union.has(slug)` — a slug
 * that does not appear in either set is a fabrication, not a real
 * resolvable doc.
 *
 * Mutates `judgments` in place. Slug-less citations are left with
 * `hallucinated` undefined since the lookup key is the slug per the
 * GraderJudgment domain doc-comment ("slug does not resolve against the
 * task's contextDocs set").
 *
 * Called from `gap-analysis-step` (literacy mode only) after
 * `descToDocRefs` and `documentManifest` are built — both inputs are
 * unavailable at extract time. Non-literacy modes skip this step
 * entirely (the runtime contract has no canonical doc set to check
 * against).
 *
 * @param judgments - Grader judgments to enrich. Mutated in place.
 * @param taskDocSlugs - Map from base task description (the form judgment
 *   `taskId` carries after stripping the `(gold)` / `(baseline)` suffix)
 *   to the slugs declared in that task's `context.docs`.
 * @param manifestSlugs - All slugs in the run's document manifest.
 */
export declare function populateHallucinationFields(judgments: GraderJudgment[], taskDocSlugs: Map<string, string[]>, manifestSlugs: Iterable<string>): void;
/**
 * Per-variant scoring profiles passed to {@link extractStoredTestResults}.
 * Each profile maps dimension id → weight. Variants whose dimensions don't
 * intersect the supplied keys yield `compositeScore: undefined` rather than
 * a misleading 0.
 */
export type StoredTestResultProfiles = Partial<Record<"gold" | "baseline", Record<string, number>>>;
/**
 * Extract per-test results with model output from evaluation results.
 *
 * Mirrors extractGraderJudgments() but captures the full StoredTestResult
 * shape including response.output (truncated), latency, and cost.
 * One StoredTestResult per test × model combination.
 *
 * When `profiles` is provided, each entry's `compositeScore` is computed as
 * the weighted mean of its dimension scores using the profile matching its
 * detected `variant`. Without profiles, `compositeScore` is omitted — legacy
 * behavior preserved.
 *
 * See D0029 and docs/design-docs/score-drill-down.md (Phase 1).
 */
export declare function extractStoredTestResults(resultsPath: string, profiles?: StoredTestResultProfiles): StoredTestResult[];
/**
 * Reconstruct the `feature → ActualScoreEntry` map that {@link mergeScores}
 * expects from an inherited agentic contribution (W-agentic-testresults-merge).
 *
 * Used on a FULL-mode partial cache hit, where the agentic sub-run was served
 * from the remote cache so `eval-results-agentic.json` is absent locally and
 * `scoreAgenticResults` cannot run. The agentic `actualScore` already lives on
 * the cached summary's per-feature scores; agentic cost + row count are summed
 * from the inherited agentic rows. Feeding this into `mergeScores` overlays the
 * agentic actual onto the freshly-computed baseline floor/ceiling — recomputing
 * retrieval gap and infrastructure efficiency against the fresh ceiling — so the
 * merge stays consistent with the file-present FULL path.
 *
 * `codeCorrectness` / `docCoverage` / `taskCompletion` mirror the cached score
 * (the agentic-specific dimension breakdown is not persisted on the summary);
 * `mergeScores` only reads them on its agentic-only branch, which cannot fire
 * here because every feature has a fresh baseline score.
 */
export declare function agenticScoresFromInherited(inheritedScores: readonly FeatureScore[], inheritedTestResults: readonly StoredTestResult[]): Record<string, ActualScoreEntry>;
/**
 * W0198 — aggregate every per-test `SymbolPreflightReport` into a single
 * resolver-health summary. Returns `undefined` when the run had no
 * preflight reports (manifest disabled, resolver missing, or every
 * candidate output cited zero in-scope packages) so the consumer can
 * cleanly omit the field from the score summary instead of writing a
 * vacuous block of zeros.
 *
 * Exported for the dedicated unit test in `preflight-summary.test.ts`;
 * production calls go through `calculateAndWriteScores`, which threads
 * the result into the `EvalScoreSummary.preflight` field.
 */
export declare function summarizePreflight(reports: Map<string, SymbolPreflightReport> | undefined): PreflightSummary | undefined;
/**
 * Aggregate per-test `ConsumerVerificationReport`s into a single roll-up —
 * `undefined` when there are none, so the summary block can be omitted.
 * Exported for the unit test in `verification-summary.test.ts`.
 */
export declare function summarizeConsumerVerification(reports: Map<string, ConsumerVerificationReport> | undefined): ConsumerVerificationSummary | undefined;
/**
 * Score agent-harness evaluation results. Unlike literacy mode, agent-harness
 * tasks don't have a with-docs/without-docs split. All results are scored
 * as a single "actual" variant using the agent-harness profile.
 *
 * Groups results by task ID (extracted from the test description prefix)
 * rather than by feature area. Each group produces a FeatureScore with
 * groupType: "task".
 *
 * Literacy-specific fields (ceilingScore, floorScore, docLift, docQualityGap)
 * are set to 0 for backward compatibility with downstream consumers.
 */
export declare function scoreAgentHarnessResults(results: TestResult[], profile: Record<string, number>, scoringOptions?: ScoreTestGroupOptions): FeatureScore[];
/**
 * Score knowledge-probe evaluation results.
 *
 * Knowledge-probe mode evaluates parametric recall: the model has no `docs`
 * var and answers from training-data knowledge alone. The compiler explicitly
 * deletes `vars.docs`, so every result lands in the without-docs bucket of
 * the literacy scoring path — collapsing testCount and ceilingScore to zero.
 *
 * This branch mirrors the shape of `scoreAgentHarnessResults` but groups by
 * feature area (KP results carry `__featureArea` from the compiler), and
 * uses the `knowledge-probe` profile (factual-correctness / completeness /
 * currency). Literacy-specific fields (ceilingScore, floorScore, docLift,
 * docQualityGap) are zero — KP has no with-docs/without-docs decomposition.
 */
export declare function scoreKnowledgeProbeResults(results: TestResult[], profile: Record<string, number>, scoringOptions?: ScoreTestGroupOptions): FeatureScore[];
/**
 * Score non-literacy results grouped by feature area on an arbitrary
 * dimension profile, zeroing the literacy-only fields (ceilingScore,
 * floorScore, docLift, docQualityGap) since these modes have no
 * with-docs/without-docs decomposition. Shared by knowledge-probe and
 * mcp-server — both classify the grader's `llm-rubric` componentResults to
 * their profile's dimensions via `scoreTestGroup` and report a feature-level
 * composite. The feature key comes from `vars.__featureArea` (set by the
 * compiler), falling back to `detectFeatureArea` on the description.
 */
export declare function scoreResultsByFeatureArea(results: TestResult[], profile: Record<string, number>, scoringOptions?: ScoreTestGroupOptions): FeatureScore[];
/**
 * Score agentic evaluation results. In agentic mode, all test entries are
 * gold-only (no baseline entries — the .expanded.agentic.yaml fix ensures this).
 * The model retrieves docs via tools, so all results map to "actual" scores.
 *
 * Returns a record keyed by feature area with the composite actual score.
 */
export declare function scoreAgenticResults(resultsPath: string, profile: Record<string, number>, scoringOptions?: ScoreTestGroupOptions): Record<string, ActualScoreEntry>;
/**
 * Score agentic results broken down by model.
 *
 * Same logic as `scoreAgenticResults` but groups by provider first,
 * producing a map of model → feature → ActualScoreEntry.
 * Used to enrich the per-model breakdown with actual scores in full mode.
 */
export declare function scoreAgenticResultsPerModel(resultsPath: string, profile: Record<string, number>, scoringOptions?: ScoreTestGroupOptions): Record<string, Record<string, ActualScoreEntry>>;
/** Options for the calculate-scores main() function. */
/**
 * Pre-built runner closure for the GRAD-04 borderline-consensus pass.
 *
 * The composition root produces one of these via
 * `createBorderlineConsensusRunner` so the threshold + replication
 * defaults stay co-located with the rest of the pipeline wiring.
 * `calculateAndWriteScores` invokes it after `extractGraderJudgments`
 * and before persisting `grader-judgments.json` so the judgments file
 * carries the consensus-merged scores rather than the original
 * single-replica grader output (CR-01).
 */
export type BorderlineConsensusRunner = (args: {
    judgments: GraderJudgment[];
    logger?: Logger;
    regrade: (judgment: GraderJudgment) => Promise<number>;
}) => Promise<{
    consistencyByJudgment: Map<string, JudgmentConsistency>;
    judgments: GraderJudgment[];
}>;
export interface CalculateScoresOptions {
    /** Allowed origins for source isolation reporting */
    allowedOrigins?: string[];
    /**
     * Pre-built borderline-consensus runner (CR-01). When provided AND
     * non-zero `borderlineReplications`, runs after extraction and
     * persists `borderline-consistency.json` alongside
     * `grader-judgments.json`. When omitted, the pipeline keeps the
     * single-replica scores — preserving Phase 2 behavior for callers
     * that haven't opted in.
     */
    borderlineConsensusRunner?: BorderlineConsensusRunner;
    /**
     * Optional regrade entry point used by the borderline runner. Wired
     * in by the orchestration step from grader-api.ts; when absent, the
     * runner is skipped because there's nothing to call.
     */
    borderlineRegradeOnce?: (responseText: string, rubricText: string) => Promise<null | number>;
    /** Logger instance (defaults to ConsoleLogger if not provided) */
    logger?: Logger;
    /** Evaluation mode (controls which result files are read) */
    mode?: string;
    /**
     * W0198 — symbol-preflight reports keyed by `${runId}/${mode}/${task}/${model}`,
     * populated by `RunEvalStep` via `emitSymbolPreflight`. When provided, the
     * scoring engine merges deterministic preflight findings into the
     * `code-correctness` dimension. Absence (or empty map) collapses cleanly
     * to the pre-W0198 rubric-only path.
     */
    preflightReports?: Map<string, SymbolPreflightReport>;
    /**
     * W0198 — preflight's share of `code-correctness` in `[0, 1]`. Defaults
     * to `DEFAULT_PREFLIGHT_CODE_CORRECTNESS_WEIGHT` when omitted.
     */
    preflightWeight?: number;
    /** Pre-resolved source config (skips loadSource() call) */
    resolvedSource?: ResolvedSourceConfig;
    /** Path to baseline results file (default: results/latest/eval-results.json) */
    resultsPath?: string;
    /** Root directory of the eval package (required) */
    rootDir: string;
    /**
     * W0198 — runId axis used to look up preflight reports. Required when
     * `preflightReports` is provided; otherwise the lookup callback can't
     * reconstruct the right key.
     */
    runId?: string;
    /** Search mode for source verification metadata */
    searchMode?: string;
    /** Documentation source name */
    source?: string;
    /**
     * Consumer-verification reports keyed by `${runId}/${mode}/${task}/${model}`,
     * populated by `RunEvalStep`; verdicts merge into `code-correctness` at
     * scoring. Absent or empty collapses to the no-verification path.
     */
    consumerVerificationReports?: Map<string, ConsumerVerificationReport>;
    /**
     * Verification's share of `code-correctness` in `[0, 1]`. Defaults to
     * `DEFAULT_CONSUMER_VERIFICATION_CODE_CORRECTNESS_WEIGHT` when omitted.
     */
    consumerVerificationWeight?: number;
    /**
     * Eval modes served from the remote cache this run (from
     * `state.remoteCacheHits`). On a partial cache hit some modes are cached and
     * some ran fresh; the COST SUMMARY notes which were inherited so the printed
     * cost is not misread as fully this-run spend. A full cache hit skips this
     * step entirely, so the block never prints in that case
     * (W-cache-provenance-notifications).
     */
    cachedModes?: readonly string[];
    /**
     * Agentic contribution inherited from a cached report on a FULL-mode
     * partial cache hit (W-agentic-testresults-merge).
     *
     * A FULL literacy run scores from two files: `eval-results.json` (baseline)
     * and `eval-results-agentic.json` (agentic). When the agentic sub-run is
     * served from the remote cache, its raw results file is never written to
     * local disk — only the artifact manifest + the cached summary are restored.
     * Without this, the re-score silently drops the agentic rows: the report
     * advertises agentic rawResults/traces but `summary.testResults` has no
     * agentic rows, so those traces are unreachable through the row drill-down.
     *
     * When present (and the local agentic results file is absent), the scorer
     * merges these inherited agentic rows + per-feature `actualScore` back into
     * the FULL-mode summary instead of degrading to baseline. Sourced from the
     * cached report's summary by `CalculateScoresStep`.
     */
    inheritedAgentic?: InheritedAgenticContribution;
}
/**
 * The agentic slice of a cached FULL-mode report, inherited on a partial
 * cache hit. See {@link CalculateScoresOptions.inheritedAgentic}.
 */
export interface InheritedAgenticContribution {
    /** Feature-level agent behavior aggregates from the cached summary (may be null). */
    agentBehavior: FeatureAgentBehavior[] | null;
    /**
     * Per-feature scores from the cached FULL summary — carries the agentic
     * `actualScore` overlaid onto the freshly-computed baseline floor/ceiling.
     */
    scores: FeatureScore[];
    /** Agentic test-result rows (modelId starts with `agentic:`) from the cached summary. */
    testResults: StoredTestResult[];
}
/** Result from calculateAndWriteScores — replaces process.exit() calls. */
export interface CalculateScoresResult {
    /** Feature areas that scored below the critical threshold (40). */
    belowCritical: string[];
    /** Summary of test execution outcomes (total, passed, failed, errored). */
    testSummary?: TestSummary;
}
export declare function calculateAndWriteScores(options: CalculateScoresOptions): Promise<CalculateScoresResult>;
/**
 * Compute a typed {@link ErrorSummary} from a raw Promptfoo results file
 * (eval failure observability, slice B1).
 *
 * An errored result is one with no `gradingResult` — the model output (if any)
 * was discarded before grading. Each errored result's `error` string is run
 * through the pure `classifyErrorString` classifier (config-error vs
 * provider-error vs unknown), replacing the historical blanket `api-error` tag.
 *
 * A task is `fullyErrored` when EVERY model × variant expansion of it errored.
 * The `(gold)` / `(baseline)` variant suffix is collapsed via `splitTaskVariant`
 * so a task's gold and baseline halves across all models are grouped together.
 *
 * This does NOT touch the headline score or its `testCount` — it is a
 * visibility signal on the summary, computed once per scoring path and spread
 * through the slim `ReportSummary` unchanged (so it rides along on cache hits).
 */
export declare function computeErrorSummary(resultsPath: string): ErrorSummary;
/**
 * One-line stdout note for the COST SUMMARY when a partial cache hit served
 * some eval modes from a prior run. Returns null when nothing was cache-served.
 * (A fully-cached run skips the calculate-scores step entirely — see
 * `calculate-scores-step.ts` — so the COST SUMMARY block never prints in that
 * case; this note covers only the partial-cache path.) Exported for testing.
 */
export declare function formatCachedModesNote(cachedModes: readonly string[] | undefined): null | string;
