export type BenchmarkTier = 'canned' | 'stress' | 'adversarial' | 'regression';
export type FailureCategory = 'WRONG_ANSWER' | 'PARTIAL_COMPLETION' | 'DID_NOT_RUN_TOOLS' | 'HALLUCINATED_API' | 'HALLUCINATED_FILE' | 'OVER_EDITED' | 'UNDER_EDITED' | 'TIMEOUT' | 'TOOL_LOOP' | 'REFUSAL_WRONG' | 'REFUSAL_CORRECT' | 'FORMAT_FAIL' | 'NO_EVIDENCE' | 'ERROR';
export interface BenchmarkTask {
    id: string;
    tier: BenchmarkTier;
    description: string;
    /** The user message text sent to /api/chat. */
    input: string;
    /** Substrings the response MUST contain (case-insensitive). */
    expectIncludes?: string[];
    /** Substrings the response MUST NOT contain (case-insensitive). */
    expectMissing?: string[];
    /** Tool names that MUST have been called at least once. */
    requireTools?: string[];
    /** Tool names that MUST NOT be called. */
    forbiddenTools?: string[];
    /**
     * Custom scorer: given the text response and tool calls, returns a
     * pass/fail verdict with a reason. Runs after include/missing checks.
     */
    customScorer?: (response: string, toolCalls: string[]) => {
        pass: boolean;
        reason: string;
    };
    /** Tags for filtering. */
    tags?: string[];
    /** Human-readable definition of what "done" means for this task. */
    definitionOfDone?: string;
    /** Explicit failure conditions beyond generic scorer checks. */
    failConditions?: string[];
    /** Tools the agent is allowed to use. When set, any other tool call fails the task. */
    toolsAllowed?: string[];
    /** Maximum turns the agent may take before the task is considered timed-out. */
    maxTurns?: number;
    /** Expected output artifacts (file paths, format types, etc). */
    expectedArtifacts?: string[];
    /** Difficulty rating for reporting purposes (1–5). */
    difficulty?: number;
}
export interface BenchmarkTaskResult {
    taskId: string;
    tier: BenchmarkTier;
    description: string;
    status: 'pass' | 'fail' | 'error';
    failureCategory?: FailureCategory;
    reason: string;
    responsePreview: string;
    toolCalls: string[];
    durationMs: number;
    tags: string[];
}
export interface BenchmarkRun {
    id: string;
    startedAt: string;
    finishedAt: string;
    model: string;
    baseUrl: string;
    tiers: BenchmarkTier[];
    total: number;
    passed: number;
    failed: number;
    errored: number;
    passRate: number;
    /** Ratio of passes to cost (approximated as total turns — real cost tracking is Gap #5). */
    passTurnRatio?: number;
    results: BenchmarkTaskResult[];
}
export interface BenchmarkRunOptions {
    /** Daemon base URL. Default http://127.0.0.1:4300. */
    baseUrl?: string;
    /** Model to use. */
    model?: string;
    /** Override fetch (for tests). */
    fetchImpl?: typeof fetch;
    /** Task set; defaults to BUILT_IN_TASKS. */
    tasks?: BenchmarkTask[];
    /** Which tiers to run; defaults to all. */
    tiers?: BenchmarkTier[];
    /** Specific task IDs to run (overrides tiers filter). */
    filterIds?: string[];
    /** Per-task timeout in ms. Default 60_000. */
    perTaskTimeoutMs?: number;
    /**
     * Project directory for persisting run results and regression tasks.
     * When set, results are written to <projectDir>/.harness/benchmarks/.
     */
    projectDir?: string;
}
/** Select tasks from the given set filtered by options. */
export declare function selectTasks(options: BenchmarkRunOptions, allTasks: BenchmarkTask[]): BenchmarkTask[];
/**
 * Run a single benchmark task against the daemon.
 */
export declare function runBenchmarkTask(task: BenchmarkTask, options?: BenchmarkRunOptions): Promise<BenchmarkTaskResult>;
/**
 * Run all selected tasks sequentially and return a structured run record.
 */
export declare function runBenchmark(options?: BenchmarkRunOptions): Promise<BenchmarkRun>;
export declare function persistBenchmarkRun(projectDir: string, run: BenchmarkRun): Promise<void>;
export declare function loadBenchmarkRuns(projectDir: string): Promise<BenchmarkRun[]>;
/** Load regression tasks that were auto-generated from past failures. */
export declare function loadRegressionTasks(projectDir?: string): Promise<BenchmarkTask[]>;
/** Summary table for display: counts per tier. */
export declare function summarizeByTier(run: BenchmarkRun): Array<{
    tier: string;
    total: number;
    passed: number;
    passRate: string;
}>;
//# sourceMappingURL=benchmark.d.ts.map