import { generateTextWithModelId } from "../llm/generate-text.js";
function classifyOpenRouterRateLimit(message) {
    const normalized = message.toLowerCase();
    if (!normalized.includes("rate limit exceeded"))
        return null;
    if (normalized.includes("per-day") ||
        normalized.includes("per day") ||
        normalized.includes("free-models-per-day")) {
        return "perDay";
    }
    if (normalized.includes("per-min") ||
        normalized.includes("per min") ||
        normalized.includes("free-models-per-min")) {
        return "perMin";
    }
    return "perMin";
}
export function classifyBenchmarkFailure(message) {
    const normalized = message.toLowerCase();
    if (normalized.includes("empty summary"))
        return "empty";
    const rateLimit = classifyOpenRouterRateLimit(message);
    if (rateLimit === "perMin")
        return "rateLimitMin";
    if (rateLimit === "perDay")
        return "rateLimitDay";
    if (normalized.includes("no allowed providers are available"))
        return "noProviders";
    if (normalized.includes("timed out") ||
        normalized.includes("timeout") ||
        normalized.includes("aborted")) {
        return "timeout";
    }
    if (normalized.includes("provider returned error") || normalized.includes("provider error")) {
        return "providerError";
    }
    return "other";
}
async function mapWithConcurrency(items, concurrency, fn) {
    const limit = Math.max(1, Math.floor(concurrency));
    const results = new Array(items.length);
    let nextIndex = 0;
    const worker = async () => {
        while (true) {
            const current = nextIndex;
            nextIndex += 1;
            if (current >= items.length)
                return;
            results[current] = await fn(items[current], current);
        }
    };
    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
    return results;
}
function createApiKeys(openrouterKey) {
    return {
        xaiApiKey: null,
        openaiApiKey: null,
        googleApiKey: null,
        anthropicApiKey: null,
        openrouterApiKey: openrouterKey,
    };
}
function buildModelResult(model, latencyMs) {
    return {
        openrouterModelId: model.id,
        initialLatencyMs: latencyMs,
        medianLatencyMs: latencyMs,
        totalLatencyMs: latencyMs,
        successCount: 1,
        contextLength: model.contextLength,
        maxCompletionTokens: model.maxCompletionTokens,
        supportedParametersCount: model.supportedParametersCount,
        modality: model.modality,
        inferredParamB: model.inferredParamB,
    };
}
async function runProbe({ modelId, apiKeys, timeoutMs, fetchImpl, generate, }) {
    await generate({
        modelId: `openai/${modelId}`,
        apiKeys,
        prompt: { userText: "Reply with a single word: OK" },
        temperature: 0,
        maxOutputTokens: 16,
        timeoutMs,
        fetchImpl,
        forceOpenRouter: true,
        retries: 0,
    });
}
export async function benchmarkOpenRouterModels({ models, openrouterKey, concurrency, timeoutMs, fetchImpl, cooldownMs = 65_000, generate = generateTextWithModelId, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), onProgress, onSuccess, onFailure, onCooldown, }) {
    const failureCounts = {
        empty: 0,
        rateLimitMin: 0,
        rateLimitDay: 0,
        noProviders: 0,
        timeout: 0,
        providerError: 0,
        other: 0,
    };
    const apiKeys = createApiKeys(openrouterKey);
    const startedAt = now();
    let done = 0;
    let okCount = 0;
    let cooldownUntilMs = 0;
    let cooldownNotifiedAtMs = 0;
    const reportProgress = () => onProgress?.({
        done,
        total: models.length,
        okCount,
        elapsedMs: now() - startedAt,
    });
    const waitForCooldown = async () => {
        const current = now();
        if (cooldownUntilMs <= current)
            return;
        const remaining = cooldownUntilMs - current;
        if (current - cooldownNotifiedAtMs > 5_000) {
            cooldownNotifiedAtMs = current;
            onCooldown?.(remaining);
        }
        await sleep(remaining);
    };
    const results = await mapWithConcurrency(models, concurrency, async (model) => {
        const runStartedAt = now();
        try {
            await waitForCooldown();
            await runProbe({
                modelId: model.id,
                apiKeys,
                timeoutMs,
                fetchImpl,
                generate,
            });
            const latencyMs = now() - runStartedAt;
            done += 1;
            okCount += 1;
            reportProgress();
            onSuccess?.({ modelId: model.id, latencyMs });
            return buildModelResult(model, latencyMs);
        }
        catch (error) {
            const message = error instanceof Error ? error.message : String(error);
            const kind = classifyBenchmarkFailure(message);
            failureCounts[kind] += 1;
            if (kind === "rateLimitMin") {
                cooldownUntilMs = Math.max(cooldownUntilMs, now() + cooldownMs);
                await waitForCooldown();
                try {
                    const retryStartedAt = now();
                    await runProbe({
                        modelId: model.id,
                        apiKeys,
                        timeoutMs,
                        fetchImpl,
                        generate,
                    });
                    const latencyMs = now() - retryStartedAt;
                    done += 1;
                    okCount += 1;
                    reportProgress();
                    onSuccess?.({ modelId: model.id, latencyMs });
                    return buildModelResult(model, latencyMs);
                }
                catch {
                    // Keep the original rate-limit failure for reporting.
                }
            }
            done += 1;
            reportProgress();
            onFailure?.({ modelId: model.id, kind, message });
            return null;
        }
    });
    return {
        working: results
            .filter((result) => Boolean(result))
            .sort((a, b) => a.medianLatencyMs - b.medianLatencyMs),
        testedCount: results.length,
        failureCounts,
    };
}
export function selectBenchmarkCandidates(working, { smart, maxCandidates, }) {
    const smartFirst = working.slice().sort((a, b) => {
        const aContext = a.contextLength ?? -1;
        const bContext = b.contextLength ?? -1;
        if (aContext !== bContext)
            return bContext - aContext;
        const aOut = a.maxCompletionTokens ?? -1;
        const bOut = b.maxCompletionTokens ?? -1;
        if (aOut !== bOut)
            return bOut - aOut;
        if (a.supportedParametersCount !== b.supportedParametersCount) {
            return b.supportedParametersCount - a.supportedParametersCount;
        }
        if (a.successCount !== b.successCount)
            return b.successCount - a.successCount;
        if (a.medianLatencyMs !== b.medianLatencyMs) {
            return a.medianLatencyMs - b.medianLatencyMs;
        }
        return a.openrouterModelId.localeCompare(b.openrouterModelId);
    });
    const fastFirst = working.slice().sort((a, b) => {
        if (a.successCount !== b.successCount)
            return b.successCount - a.successCount;
        if (a.medianLatencyMs !== b.medianLatencyMs) {
            return a.medianLatencyMs - b.medianLatencyMs;
        }
        return a.openrouterModelId.localeCompare(b.openrouterModelId);
    });
    const picked = new Set();
    const ordered = [];
    for (const model of smartFirst) {
        if (ordered.length >= Math.min(smart, maxCandidates))
            break;
        if (picked.has(model.openrouterModelId))
            continue;
        picked.add(model.openrouterModelId);
        ordered.push(model.openrouterModelId);
    }
    for (const model of fastFirst) {
        if (ordered.length >= maxCandidates)
            break;
        if (picked.has(model.openrouterModelId))
            continue;
        picked.add(model.openrouterModelId);
        ordered.push(model.openrouterModelId);
    }
    return ordered;
}
export async function refineBenchmarkCandidates({ working, selectedIds, extraRuns, openrouterKey, timeoutMs, fetchImpl, generate = generateTextWithModelId, now = Date.now, onFailure, }) {
    const refined = working.slice();
    if (extraRuns <= 0 || selectedIds.length === 0)
        return refined;
    const apiKeys = createApiKeys(openrouterKey);
    const byId = new Map(refined.map((model) => [model.openrouterModelId, model]));
    for (const modelId of selectedIds) {
        const entry = byId.get(modelId);
        if (!entry)
            continue;
        const latencies = [entry.initialLatencyMs];
        let successCount = entry.successCount;
        let lastError = null;
        for (let run = 0; run < extraRuns; run += 1) {
            const runStartedAt = now();
            try {
                await runProbe({ modelId, apiKeys, timeoutMs, fetchImpl, generate });
                successCount += 1;
                const latencyMs = now() - runStartedAt;
                entry.totalLatencyMs += latencyMs;
                latencies.push(latencyMs);
            }
            catch (error) {
                lastError = error;
            }
        }
        if (successCount === 0 && lastError) {
            onFailure?.({ modelId, error: lastError });
            continue;
        }
        latencies.sort((a, b) => a - b);
        entry.medianLatencyMs = latencies[Math.floor(latencies.length / 2)] ?? entry.medianLatencyMs;
        entry.successCount = successCount;
    }
    return refined;
}
//# sourceMappingURL=benchmark.js.map