import type logger from './logger';
import type { fetchWithCache, getCache } from './cache';
export type FilePath = string;
export interface CommandLineOptions {
    prompts?: FilePath[];
    providers: FilePath[];
    output: FilePath[];
    maxConcurrency: string;
    repeat: string;
    delay: string;
    vars?: FilePath;
    tests?: FilePath;
    config?: FilePath[];
    assertions?: FilePath;
    modelOutputs?: FilePath;
    verbose?: boolean;
    grader?: string;
    tableCellMaxLength?: string;
    write?: boolean;
    cache?: boolean;
    table?: boolean;
    share?: boolean;
    progressBar?: boolean;
    watch?: boolean;
    interactiveProviders?: boolean;
    filterFailing?: string;
    filterFirstN?: string;
    filterPattern?: string;
    var?: Record<string, string>;
    generateSuggestions?: boolean;
    promptPrefix?: string;
    promptSuffix?: string;
    envFile?: FilePath;
}
export interface EnvOverrides {
    ANTHROPIC_API_KEY?: string;
    BAM_API_KEY?: string;
    BAM_API_HOST?: string;
    AZURE_OPENAI_API_HOST?: string;
    AZURE_OPENAI_API_KEY?: string;
    AZURE_OPENAI_API_BASE_URL?: string;
    AZURE_OPENAI_BASE_URL?: string;
    AWS_BEDROCK_REGION?: string;
    COHERE_API_KEY?: string;
    OPENAI_API_KEY?: string;
    OPENAI_API_HOST?: string;
    OPENAI_API_BASE_URL?: string;
    OPENAI_BASE_URL?: string;
    OPENAI_ORGANIZATION?: string;
    REPLICATE_API_KEY?: string;
    REPLICATE_API_TOKEN?: string;
    LOCALAI_BASE_URL?: string;
    MISTRAL_API_HOST?: string;
    MISTRAL_API_BASE_URL?: string;
    PALM_API_KEY?: string;
    PALM_API_HOST?: string;
    GOOGLE_API_KEY?: string;
    GOOGLE_API_HOST?: string;
    VERTEX_API_KEY?: string;
    VERTEX_API_HOST?: string;
    VERTEX_PROJECT_ID?: string;
    VERTEX_REGION?: string;
    VERTEX_PUBLISHER?: string;
    MISTRAL_API_KEY?: string;
    CLOUDFLARE_API_KEY?: string;
    CLOUDFLARE_ACCOUNT_ID?: string;
}
export interface ProviderOptions {
    id?: ProviderId;
    label?: ProviderLabel;
    config?: any;
    prompts?: string[];
    transform?: string;
    delay?: number;
    env?: EnvOverrides;
}
export declare function isProviderOptions(provider: any): provider is ProviderOptions;
export interface CallApiContextParams {
    vars: Record<string, string | object>;
    logger?: typeof logger;
    fetchWithCache?: typeof fetchWithCache;
    getCache?: typeof getCache;
}
export interface CallApiOptionsParams {
    includeLogProbs?: boolean;
    originalProvider?: ApiProvider;
}
type CallApiFunction = {
    (prompt: string, context?: CallApiContextParams, options?: CallApiOptionsParams): Promise<ProviderResponse>;
    label?: string;
};
export interface ApiProvider {
    id: () => string;
    callApi: CallApiFunction;
    callEmbeddingApi?: (prompt: string) => Promise<ProviderEmbeddingResponse>;
    callClassificationApi?: (prompt: string) => Promise<ProviderClassificationResponse>;
    label?: ProviderLabel;
    transform?: string;
    delay?: number;
}
export declare function isApiProvider(provider: any): provider is ApiProvider;
export interface ApiEmbeddingProvider extends ApiProvider {
    callEmbeddingApi: (input: string) => Promise<ProviderEmbeddingResponse>;
}
export interface ApiSimilarityProvider extends ApiProvider {
    callSimilarityApi: (reference: string, input: string) => Promise<ProviderSimilarityResponse>;
}
export interface ApiClassificationProvider extends ApiProvider {
    callClassificationApi: (prompt: string) => Promise<ProviderClassificationResponse>;
}
export interface ApiModerationProvider extends ApiProvider {
    callModerationApi: (prompt: string, response: string) => Promise<ProviderModerationResponse>;
}
export interface TokenUsage {
    total: number;
    prompt: number;
    completion: number;
    cached?: number;
}
export interface ProviderResponse {
    error?: string;
    output?: string | object;
    tokenUsage?: Partial<TokenUsage>;
    cost?: number;
    cached?: boolean;
    logProbs?: number[];
}
export interface ProviderEmbeddingResponse {
    error?: string;
    embedding?: number[];
    tokenUsage?: Partial<TokenUsage>;
}
export interface ProviderSimilarityResponse {
    error?: string;
    similarity?: number;
    tokenUsage?: Partial<TokenUsage>;
}
export interface ProviderClassificationResponse {
    error?: string;
    classification?: Record<string, number>;
}
export interface ModerationFlag {
    code: string;
    description: string;
    confidence: number;
}
export interface ProviderModerationResponse {
    error?: string;
    flags?: ModerationFlag[];
}
export interface CsvRow {
    [key: string]: string;
}
export type VarMapping = Record<string, string>;
export type ProviderType = 'embedding' | 'classification' | 'text' | 'moderation';
export type ProviderTypeMap = Partial<Record<ProviderType, string | ProviderOptions | ApiProvider>>;
export interface GradingConfig {
    rubricPrompt?: string | string[];
    provider?: string | ProviderOptions | ApiProvider | ProviderTypeMap;
    factuality?: {
        subset?: number;
        superset?: number;
        agree?: number;
        disagree?: number;
        differButFactual?: number;
    };
}
export interface PromptConfig {
    prefix?: string;
    suffix?: string;
}
export interface OutputConfig {
    /**
     * @deprecated in > 0.38.0. Use `transform` instead.
     */
    postprocess?: string;
    transform?: string;
    storeOutputAs?: string;
}
export interface RunEvalOptions {
    provider: ApiProvider;
    prompt: Prompt;
    delay: number;
    test: AtomicTestCase;
    nunjucksFilters?: NunjucksFilterMap;
    evaluateOptions: EvaluateOptions;
    rowIndex: number;
    colIndex: number;
    repeatIndex: number;
}
export interface EvaluateOptions {
    maxConcurrency?: number;
    showProgressBar?: boolean;
    progressCallback?: (progress: number, total: number, index: number, evalStep: RunEvalOptions) => void;
    generateSuggestions?: boolean;
    repeat?: number;
    delay?: number;
    cache?: boolean;
    eventSource?: string;
    interactiveProviders?: boolean;
}
export interface Prompt {
    id?: string;
    raw: string;
    /**
     * @deprecated in > 0.59.0. Use `label` instead.
     */
    display?: string;
    label: string;
    function?: (context: {
        vars: Record<string, string | object>;
        provider?: ApiProvider;
    }) => Promise<string | object>;
}
export type CompletedPrompt = Prompt & {
    provider: string;
    metrics?: {
        score: number;
        testPassCount: number;
        testFailCount: number;
        assertPassCount: number;
        assertFailCount: number;
        totalLatencyMs: number;
        tokenUsage: TokenUsage;
        namedScores: Record<string, number>;
        cost: number;
    };
};
export interface PromptWithMetadata {
    id: string;
    prompt: Prompt;
    recentEvalDate: Date;
    recentEvalId: string;
    evals: {
        id: string;
        datasetId: string;
        metrics: CompletedPrompt['metrics'];
    }[];
    count: number;
}
export interface EvaluateResult {
    provider: Pick<ProviderOptions, 'id' | 'label'>;
    prompt: Prompt;
    vars: Record<string, string | object>;
    response?: ProviderResponse;
    error?: string;
    success: boolean;
    score: number;
    latencyMs: number;
    gradingResult?: GradingResult;
    namedScores: Record<string, number>;
    cost?: number;
}
export interface EvaluateTableOutput {
    pass: boolean;
    score: number;
    namedScores: Record<string, number>;
    text: string;
    prompt: string;
    latencyMs: number;
    provider?: string;
    tokenUsage?: Partial<TokenUsage>;
    gradingResult?: GradingResult;
    cost: number;
}
export interface EvaluateTableRow {
    description?: string;
    outputs: EvaluateTableOutput[];
    vars: string[];
    test: AtomicTestCase;
}
export interface EvaluateTable {
    head: {
        prompts: CompletedPrompt[];
        vars: string[];
    };
    body: EvaluateTableRow[];
}
export interface EvaluateStats {
    successes: number;
    failures: number;
    tokenUsage: Required<TokenUsage>;
}
export interface EvaluateSummary {
    version: number;
    timestamp: string;
    results: EvaluateResult[];
    table: EvaluateTable;
    stats: EvaluateStats;
}
export interface GradingResult {
    pass: boolean;
    score: number;
    reason: string;
    namedScores?: Record<string, number>;
    tokensUsed?: TokenUsage;
    componentResults?: GradingResult[];
    assertion?: Assertion | null;
    comment?: string;
}
export declare function isGradingResult(result: any): result is GradingResult;
type BaseAssertionTypes = 'human' | 'equals' | 'contains' | 'icontains' | 'contains-all' | 'contains-any' | 'icontains-all' | 'icontains-any' | 'starts-with' | 'regex' | 'is-json' | 'is-sql' | 'contains-json' | 'contains-sql' | 'javascript' | 'python' | 'similar' | 'answer-relevance' | 'context-faithfulness' | 'context-recall' | 'context-relevance' | 'llm-rubric' | 'model-graded-closedqa' | 'factuality' | 'model-graded-factuality' | 'webhook' | 'rouge-n' | 'rouge-s' | 'rouge-l' | 'levenshtein' | 'is-valid-openai-function-call' | 'is-valid-openai-tools-call' | 'latency' | 'perplexity' | 'perplexity-score' | 'cost' | 'select-best' | 'moderation';
type NotPrefixed<T extends string> = `not-${T}`;
export type AssertionType = BaseAssertionTypes | NotPrefixed<BaseAssertionTypes>;
export interface AssertionSet {
    type: 'assert-set';
    assert: Assertion[];
    weight?: number;
    metric?: string;
    threshold?: number;
}
export interface Assertion {
    type: AssertionType;
    value?: AssertionValue;
    threshold?: number;
    weight?: number;
    provider?: GradingConfig['provider'];
    rubricPrompt?: GradingConfig['rubricPrompt'];
    metric?: string;
    transform?: string;
}
export type AssertionValue = string | string[] | object | AssertionValueFunction;
export type AssertionValueFunction = (output: string, context: AssertionValueFunctionContext) => AssertionValueFunctionResult | Promise<AssertionValueFunctionResult>;
export interface AssertionValueFunctionContext {
    prompt: string | undefined;
    vars: Record<string, string | object>;
    test: AtomicTestCase<Record<string, string | object>>;
}
export type AssertionValueFunctionResult = boolean | number | GradingResult;
export interface TestCasesWithMetadataPrompt {
    prompt: CompletedPrompt;
    id: string;
    evalId: string;
}
export interface TestCasesWithMetadata {
    id: string;
    testCases: FilePath | (FilePath | TestCase)[];
    recentEvalDate: Date;
    recentEvalId: string;
    count: number;
    prompts: TestCasesWithMetadataPrompt[];
}
export interface TestCase<Vars = Record<string, string | string[] | object>> {
    description?: string;
    vars?: Vars;
    provider?: string | ProviderOptions | ApiProvider;
    providerOutput?: string | object;
    assert?: (AssertionSet | Assertion)[];
    options?: PromptConfig & OutputConfig & GradingConfig & {
        disableVarExpansion?: boolean;
        disableConversationVar?: boolean;
    };
    threshold?: number;
}
export interface Scenario {
    description?: string;
    config: Partial<TestCase>[];
    tests: TestCase[];
}
export interface AtomicTestCase<Vars = Record<string, string | object>> extends TestCase {
    vars?: Record<string, string | object>;
}
export type NunjucksFilterMap = Record<string, (...args: any[]) => string>;
export type DerivedMetric = {
    name: string;
    value: string | ((scores: Record<string, number>, context: RunEvalOptions) => number);
};
export interface TestSuite {
    description?: string;
    providers: ApiProvider[];
    prompts: Prompt[];
    providerPromptMap?: Record<string, string[]>;
    tests?: TestCase[];
    scenarios?: Scenario[];
    defaultTest?: Partial<TestCase>;
    nunjucksFilters?: NunjucksFilterMap;
    env?: EnvOverrides;
    derivedMetrics?: DerivedMetric[];
}
export type ProviderId = string;
export type ProviderLabel = string;
export type ProviderFunction = ApiProvider['callApi'];
export type ProviderOptionsMap = Record<ProviderId, ProviderOptions>;
export interface TestSuiteConfig {
    description?: string;
    providers: ProviderId | ProviderFunction | (ProviderId | ProviderOptionsMap | ProviderOptions | ProviderFunction)[];
    prompts: FilePath | (FilePath | Prompt)[] | Record<FilePath, string>;
    tests: FilePath | (FilePath | TestCase)[];
    scenarios?: Scenario[];
    defaultTest?: Omit<TestCase, 'description'>;
    outputPath?: FilePath | FilePath[];
    sharing?: boolean | {
        apiBaseUrl?: string;
        appBaseUrl?: string;
    };
    nunjucksFilters?: Record<string, FilePath>;
    env?: EnvOverrides;
    derivedMetrics?: DerivedMetric[];
    metadata?: Record<string, any>;
}
export type UnifiedConfig = TestSuiteConfig & {
    evaluateOptions: EvaluateOptions;
    commandLineOptions: Partial<CommandLineOptions>;
};
export interface EvalWithMetadata {
    id: string;
    date: Date;
    config: Partial<UnifiedConfig>;
    results: EvaluateSummary;
    description?: string;
}
export type PromptFunction = (context: {
    vars: Record<string, string | object>;
}) => Promise<string | object>;
export type EvaluateTestSuite = {
    prompts: (string | object | PromptFunction)[];
    writeLatestResults?: boolean;
} & Omit<TestSuiteConfig, 'prompts'>;
export type EvaluateTestSuiteWithEvaluateOptions = EvaluateTestSuite & {
    evaluateOptions: EvaluateOptions;
};
export interface SharedResults {
    data: ResultsFile;
}
export interface ResultsFile {
    version: number;
    createdAt: string;
    results: EvaluateSummary;
    config: Partial<UnifiedConfig>;
}
export interface OutputFile {
    evalId: string | null;
    results: EvaluateSummary;
    config: Partial<UnifiedConfig>;
    shareableUrl: string | null;
}
export interface Job {
    status: 'in-progress' | 'complete';
    progress: number;
    total: number;
    result: EvaluateSummary | null;
}
export {};
//# sourceMappingURL=types.d.ts.map