import { a as RunSplitTag } from './run-record-BGY6bHRh.js';

/**
 * Shared types for the reference benchmark wrappers under
 * `src/benchmarks/`. Each wrapper exports the three functions in
 * `BenchmarkAdapter` plus its own typed `DatasetItem` shape.
 */

interface BenchmarkDatasetItem<TPayload = unknown> {
    /** Stable dataset-local item id (used for split assignment + paper
     *  references). Unique within a benchmark. */
    id: string;
    /** Free-form payload. Each benchmark defines its own shape. */
    payload: TPayload;
}
interface BenchmarkEvaluation {
    /** [0, 1] score for the response on this item. Exact-match
     *  benchmarks use 0/1; partial-credit benchmarks may return
     *  fractional values. */
    score: number;
    /** Optional bag of raw scoring signals — e.g. parsed numeric
     *  answer, regex match, judge sub-scores. */
    raw: Record<string, unknown>;
}
/** Common signature implemented by every adapter under `src/benchmarks/*`. */
interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown> {
    /** Load the dataset for the given split. May hit the network on
     *  first call but should be cache-friendly. Adapters that don't
     *  ship the dataset itself MUST throw a clearly-marked error
     *  pointing the caller at the loader script. */
    loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>;
    /** Score a single response. Pure with respect to the inputs. */
    evaluate(item: BenchmarkDatasetItem<TPayload>, response: string): Promise<BenchmarkEvaluation>;
    /** Deterministic split assignment via item id hashing. The
     *  fraction of items in each split is implementation-defined but
     *  MUST be stable across processes and platforms. */
    assignSplit(itemId: string): RunSplitTag;
}
/** Split-assignment seed shared across all benchmarks. Bumping this
 *  value reshuffles every split — do NOT do that lightly. */
declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
/**
 * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
 * stable 32-bit hash of `${seed}::${id}`. Default proportions:
 *
 *   search: 60%   (optimization-readable)
 *   dev:    20%   (held-out for tuning, leak-on-purpose during dev)
 *   holdout:20%   (paper-grade held-out, gated reads)
 */
declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;

/**
 * Synthetic routing dataset. 16 tasks across 4 categories. Used as a
 * deterministic, dependency-free benchmark for any router that maps a
 * natural-language request to one of a fixed set of route labels.
 *
 * Format (see `routing/README.md` for prose):
 *
 *   {
 *     id:           stable per-task ID (matches across processes).
 *     category:     one of the four route labels.
 *     prompt:       the user-facing request the router must classify.
 *     route:        the ground-truth route the router should pick.
 *     synonyms:     other strings that count as a correct answer.
 *     hardNegatives:close-but-wrong route labels — used to detect the
 *                   "always picks the popular route" failure mode.
 *   }
 *
 * The four categories are intentionally cross-domain (file ops,
 * math, search, conversation) so a router that collapses to one
 * category is easy to spot.
 */
interface RoutingItem {
    id: string;
    category: 'file' | 'math' | 'search' | 'chat';
    prompt: string;
    /** Canonical correct route label. */
    route: string;
    /** Alternate route labels that also count as correct. */
    synonyms: string[];
    /** Wrong-but-tempting route labels (for analysis, not grading). */
    hardNegatives: string[];
}
declare const ROUTING_DATASET: RoutingItem[];

/**
 * Routing benchmark — synthetic, dependency-free, ships in the
 * package. 16 cross-category items in `dataset.ts`. See
 * `routing/README.md` for the format.
 *
 * `evaluate` does case-insensitive exact match against the canonical
 * route plus declared synonyms. The first valid route token in the
 * response wins; everything else is ignored. Wrong answers also
 * report whether they hit a hard negative — useful when triaging
 * "always picks the popular route" failure modes.
 */

type RoutingPayload = RoutingItem;
type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>;
declare class RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {
    loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]>;
    evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation>;
    assignSplit(itemId: string): RunSplitTag;
}
/**
 * Pull route-shaped tokens out of a model response. Routes look like
 * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics
 * are not routes, but `category.action` patterns are robust to most
 * model wrappers (JSON output, prose explanations, code fences).
 */
declare function extractRouteTokens(response: string): string[];
declare const loadDataset: (split: RunSplitTag) => Promise<RoutingDatasetItem[]>;
declare const evaluate: (item: RoutingDatasetItem, response: string) => Promise<BenchmarkEvaluation>;
declare const assignSplit: (itemId: string) => RunSplitTag;

declare const index$1_ROUTING_DATASET: typeof ROUTING_DATASET;
type index$1_RoutingAdapter = RoutingAdapter;
declare const index$1_RoutingAdapter: typeof RoutingAdapter;
type index$1_RoutingDatasetItem = RoutingDatasetItem;
type index$1_RoutingItem = RoutingItem;
type index$1_RoutingPayload = RoutingPayload;
declare const index$1_assignSplit: typeof assignSplit;
declare const index$1_evaluate: typeof evaluate;
declare const index$1_extractRouteTokens: typeof extractRouteTokens;
declare const index$1_loadDataset: typeof loadDataset;
declare namespace index$1 {
  export { index$1_ROUTING_DATASET as ROUTING_DATASET, index$1_RoutingAdapter as RoutingAdapter, type index$1_RoutingDatasetItem as RoutingDatasetItem, type index$1_RoutingItem as RoutingItem, type index$1_RoutingPayload as RoutingPayload, index$1_assignSplit as assignSplit, index$1_evaluate as evaluate, index$1_extractRouteTokens as extractRouteTokens, index$1_loadDataset as loadDataset };
}

/**
 * Reference benchmark wrappers — entry point.
 *
 * Core surface (exported here):
 *   - The `BenchmarkAdapter` contract.
 *   - `deterministicSplit` + `BENCHMARK_SPLIT_SEED` for split assignment.
 *   - `routing` — synthetic 16-task router benchmark. The only novel
 *     benchmark we built; ships in the package.
 *
 * Example wrappers (under `examples/benchmarks/`, NOT in the bundle):
 *   - `gsm8k`         — exact-match math reasoning (HF mirror, dataset
 *                       not bundled).
 *   - `swebench-lite` — 30-instance SWE-Bench subset via an external
 *                       grader command.
 *
 * The example wrappers are reference implementations of `BenchmarkAdapter`.
 * Read them, copy them, adapt them. They're intentionally not in the main
 * entry — every team will configure them differently.
 */

declare const index_BENCHMARK_SPLIT_SEED: typeof BENCHMARK_SPLIT_SEED;
type index_BenchmarkAdapter<_TItem = unknown, TPayload = unknown> = BenchmarkAdapter<_TItem, TPayload>;
type index_BenchmarkDatasetItem<TPayload = unknown> = BenchmarkDatasetItem<TPayload>;
type index_BenchmarkEvaluation = BenchmarkEvaluation;
declare const index_deterministicSplit: typeof deterministicSplit;
declare namespace index {
  export { index_BENCHMARK_SPLIT_SEED as BENCHMARK_SPLIT_SEED, type index_BenchmarkAdapter as BenchmarkAdapter, type index_BenchmarkDatasetItem as BenchmarkDatasetItem, type index_BenchmarkEvaluation as BenchmarkEvaluation, index_deterministicSplit as deterministicSplit, index$1 as routing };
}

export { BENCHMARK_SPLIT_SEED as B, type BenchmarkAdapter as a, type BenchmarkDatasetItem as b, type BenchmarkEvaluation as c, deterministicSplit as d, index$1 as e, index as i };
