/**
 * src/models/failover.ts
 * مدير التحويل التلقائي — إذا فشل النموذج الأساسي، جرب البديل
 * Failover order: Qwen → Gemini → DeepSeek
 *
 * STRATEGY (locked 2026-05-04): NVIDIA NIM only through the 30-day demo phase.
 * All providers below currently route to NIM via model aliases. Switch to the
 * full 3-provider hybrid (Gemini Flash via Google + Qwen 72B via Together AI +
 * DeepSeek R1 via DeepSeek API) when EITHER:
 *   (a) the first paying customer signs, OR
 *   (b) NIM rate-limits / quality become a customer-visible blocker.
 * See ~/.claude/.../memory/al-moasis.md and muassis-strategy.md §4 for context.
 */
import { createChildLogger } from '../utils/logger.js';
import { ModelError } from './provider.js';
import { nvidia } from './nvidia.js';
import { ollama, probeOllama } from './ollama.js';
const log = createChildLogger('models:failover');
// ─── Failover Chain ──────────────────────────────────────────────────
/**
 * Phase 1 chain: NIM (bundled key or user key) is primary. Local Ollama
 * is a soft fallback appended at boot if the daemon is detected — this
 * way a NIM 400 / 500 / quota-revoke scenario quietly degrades to local
 * inference instead of bubbling "(E400)" up to the customer.
 *
 * Ollama is appended by initFailover() if probeOllama() succeeds — never
 * hard-required. Customers without Ollama keep the NIM-only behavior.
 */
const FAILOVER_CHAINS = {
    brain: [nvidia],
    flash: [nvidia],
    reasoning: [nvidia],
};
/**
 * Probe Ollama at gateway boot. Only wires it into the failover chain if
 * the operator explicitly opts in via MUASSIS_OLLAMA_FALLBACK=1. Default
 * UX is NIM-only (bundled key + qwen3-next-80b) — we don't want a mass
 * customer flow to depend on the user installing Ollama or pulling models.
 *
 * Returns the probe result either way so the boot banner can show what's
 * available locally without committing to using it.
 */
let chainExtended = false;
export async function initFailover() {
    const probe = await probeOllama();
    if (chainExtended)
        return probe;
    // DeepSeek: secondary brain provider when the user has a key. Auto-detected
    // — no env flag needed; just paste DEEPSEEK_API_KEY=... in chat or set in
    // .env. Order: NIM rate-limit → DeepSeek → Gemini → Ollama → final 5xx.
    if (process.env.DEEPSEEK_API_KEY) {
        try {
            const { deepseek } = await import('./deepseek.js');
            for (const tier of Object.keys(FAILOVER_CHAINS)) {
                FAILOVER_CHAINS[tier].push(deepseek);
            }
            log.info('DeepSeek appended to failover chain (DEEPSEEK_API_KEY detected)');
        }
        catch (err) {
            log.warn({ err: err.message }, 'Failed to attach DeepSeek provider');
        }
    }
    // Gemini Flash: tertiary fallback for fast tier when configured
    if (process.env.GOOGLE_AI_API_KEY && process.env.GOOGLE_AI_API_KEY !== 'mock_google_key') {
        try {
            const { gemini } = await import('./gemini.js');
            for (const tier of Object.keys(FAILOVER_CHAINS)) {
                FAILOVER_CHAINS[tier].push(gemini);
            }
            log.info('Gemini appended to failover chain (GOOGLE_AI_API_KEY detected)');
        }
        catch (err) {
            log.warn({ err: err.message }, 'Failed to attach Gemini provider');
        }
    }
    // Anthropic Claude — paid frontier model. PROMOTED to primary when key is
    // set (insert at index 0) because customers who pay for Anthropic want
    // their messages to actually use it, not fall back to NIM only when
    // Anthropic 5xx's. NIM stays in the chain as a free fallback.
    if (process.env.ANTHROPIC_API_KEY) {
        try {
            const { anthropic } = await import('./anthropic.js');
            for (const tier of Object.keys(FAILOVER_CHAINS)) {
                FAILOVER_CHAINS[tier].unshift(anthropic);
            }
            log.info('Anthropic Claude promoted to primary in failover chain (ANTHROPIC_API_KEY detected)');
        }
        catch (err) {
            log.warn({ err: err.message }, 'Failed to attach Anthropic provider');
        }
    }
    // OpenAI ChatGPT — paid frontier. Same priority logic as Anthropic.
    if (process.env.OPENAI_API_KEY) {
        try {
            const { openai } = await import('./openai.js');
            for (const tier of Object.keys(FAILOVER_CHAINS)) {
                FAILOVER_CHAINS[tier].unshift(openai);
            }
            log.info('OpenAI promoted to primary in failover chain (OPENAI_API_KEY detected)');
        }
        catch (err) {
            log.warn({ err: err.message }, 'Failed to attach OpenAI provider');
        }
    }
    // Kimi (Moonshot) — fallback tier when key is set. Slotted between NIM
    // and the OpenAI/Anthropic primaries — useful for very long context jobs.
    if (process.env.MOONSHOT_API_KEY) {
        try {
            const { kimi } = await import('./kimi.js');
            for (const tier of Object.keys(FAILOVER_CHAINS)) {
                // Insert after the first provider (so the user-chosen primary
                // stays first if they set both Anthropic + Kimi)
                FAILOVER_CHAINS[tier].splice(1, 0, kimi);
            }
            log.info('Kimi appended to failover chain (MOONSHOT_API_KEY detected)');
        }
        catch (err) {
            log.warn({ err: err.message }, 'Failed to attach Kimi provider');
        }
    }
    // Universal OpenAI-compatible providers — Groq, Together, Perplexity, xAI,
    // Mistral, OpenRouter, Vercel-AI, ByteDance, HuggingFace, SGLang, vLLM.
    // One adapter, 10+ providers. Each is auto-included when its env key is
    // set; appended to the END of the chain so user-chosen primaries
    // (Anthropic/OpenAI) stay first. This is the OpenClaw-parity move that
    // takes Muassis from 6 to 16+ supported providers.
    try {
        const { activeUniversalProviders } = await import('./universal.js');
        const universals = activeUniversalProviders();
        for (const uni of universals) {
            for (const tier of Object.keys(FAILOVER_CHAINS)) {
                FAILOVER_CHAINS[tier].push(uni);
            }
            log.info({ provider: uni.name, model: uni.model }, '🌐 Universal provider attached to failover chain');
        }
        if (universals.length) {
            log.info(`🌐 ${universals.length} universal OpenAI-compatible providers active`);
        }
    }
    catch (err) {
        log.warn({ err: err.message }, 'Failed to attach universal providers');
    }
    // Ollama: opt-in last-resort local model
    const optIn = process.env.MUASSIS_OLLAMA_FALLBACK === '1';
    if (probe.available && optIn) {
        for (const tier of Object.keys(FAILOVER_CHAINS)) {
            FAILOVER_CHAINS[tier].push(ollama);
        }
        log.info({ model: probe.model, installedCount: probe.installedCount }, 'Ollama opt-in active (MUASSIS_OLLAMA_FALLBACK=1) — added to failover chain');
    }
    else {
        log.debug({ available: probe.available, optIn }, 'Ollama not in failover chain');
    }
    chainExtended = true;
    return probe;
}
/**
 * Rebuild the failover chains from scratch. Called after a BYOK key is added
 * or removed at runtime (POST/DELETE /api/models/keys) so the new provider is
 * attached/promoted IMMEDIATELY — without this, initFailover()'s one-shot
 * `chainExtended` guard meant a pasted key stayed inert until a full restart
 * (the "I added my key but it still won't connect" bug). Resets each tier to
 * the base [nvidia] chain (mutating in place so callWithFailover's live
 * `FAILOVER_CHAINS[tier]` reads pick it up), then re-runs initFailover() which
 * re-reads process.env and re-attaches every configured provider.
 */
export async function rebuildFailoverChains() {
    FAILOVER_CHAINS.brain = [nvidia];
    FAILOVER_CHAINS.flash = [nvidia];
    FAILOVER_CHAINS.reasoning = [nvidia];
    chainExtended = false;
    const probe = await initFailover();
    log.info('🔁 Failover chains rebuilt after runtime key change');
    return probe;
}
// Retryable errors (429/5xx) get up to MAX_RETRIES attempts on the same
// provider before failover. Backoff is exponential with jitter to avoid
// hammering NIM in lockstep when multiple Muassis devices hit a rate-limit
// at the same instant.
//
// Bumped from 2→4 retries (and a higher cap) because the NIM proxy on
// Vercel cold-starts at ~5-8s; with only 2 short retries we'd give up
// before the cold start finished, surfacing E503 to the customer.
const MAX_RETRIES_PER_PROVIDER = 4;
const RETRY_BASE_MS = 600;
function backoffDelay(attempt) {
    const base = RETRY_BASE_MS * Math.pow(2, attempt);
    const jitter = Math.random() * RETRY_BASE_MS;
    return Math.min(base + jitter, 12_000);
}
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
/**
 * استدعاء نموذج مع تحويل تلقائي عند الفشل
 * @param tier - المستوى المطلوب: 'brain' | 'flash' | 'reasoning'
 * @param tenantId - optional. When provided, the per-tenant preference at
 *                   identity_facts.preference.model.<tier> overrides the
 *                   default chain ordering. This is what makes the dashboard
 *                   model picker actually take effect — same code path,
 *                   tenant-specific top of chain. WhatsApp / Telegram / web
 *                   chat all funnel here so a change applies everywhere.
 */
export async function callWithFailover(tier, messages, options = {}, tenantId, modelOverride) {
    // Resolve the per-tenant model preference, if any. We build a one-shot
    // chain that puts the chosen provider+model FIRST, then falls through to
    // the default chain for resilience. The chosen provider's `.model` field
    // is mutated for this call only and restored on exit.
    let chain = FAILOVER_CHAINS[tier] ?? FAILOVER_CHAINS.brain;
    let restoreModel = null;
    // An explicit modelOverride (from the automatic router — e.g. "background
    // work → Sonnet") takes precedence over the per-tenant pin. This is how we
    // keep background loops off Opus even when the owner's brain pin is Opus.
    const forced = modelOverride
        ? { provider: modelOverride.provider, model: modelOverride.model, source: 'override' }
        : null;
    if (forced || tenantId) {
        try {
            const choice = forced ?? await (async () => {
                const { resolveActiveModel } = await import('./registry.js');
                return resolveActiveModel(tenantId, tier);
            })();
            if (choice) {
                const provider = await resolveProviderInstance(choice.provider);
                if (provider) {
                    // Stash original model + override
                    const original = provider.model;
                    provider.model = choice.model;
                    restoreModel = () => { provider.model = original; };
                    // Promote this provider to top of chain (deduped; if already present, move it)
                    const others = chain.filter(p => p !== provider);
                    chain = [provider, ...others];
                    log.info({ tenantId, tier, provider: choice.provider, model: choice.model, source: choice.source }, '🎯 Per-tenant model preference applied');
                }
            }
        }
        catch (err) {
            log.warn({ err: err.message }, 'Failed to resolve tenant model preference, using default chain');
        }
    }
    const errors = [];
    try {
        for (let i = 0; i < chain.length; i++) {
            const provider = chain[i];
            for (let retry = 0; retry <= MAX_RETRIES_PER_PROVIDER; retry++) {
                try {
                    log.debug({ tier, provider: provider.name, providerAttempt: i + 1, retry }, i === 0 && retry === 0 ? 'Calling primary provider' : 'Retrying / failing over');
                    const result = await provider.chat(messages, options);
                    return {
                        ...result,
                        requestedTier: tier,
                        failedOver: i > 0,
                        errors,
                    };
                }
                catch (err) {
                    const errorMsg = err instanceof Error ? err.message : String(err);
                    const isRetryable = err instanceof ModelError && err.isRetryable;
                    errors.push({ provider: provider.name, error: `${errorMsg}${retry > 0 ? ` (retry ${retry})` : ''}` });
                    log.warn({
                        tier,
                        provider: provider.name,
                        error: errorMsg,
                        statusCode: err instanceof ModelError ? err.statusCode : 'unknown',
                        retry,
                        isRetryable,
                    }, isRetryable && retry < MAX_RETRIES_PER_PROVIDER
                        ? 'Provider returned retryable error, will back off and retry'
                        : 'Provider failed, moving to next or giving up');
                    if (isRetryable && retry < MAX_RETRIES_PER_PROVIDER) {
                        await sleep(backoffDelay(retry));
                        continue;
                    }
                    // Non-retryable, or retries exhausted — break out and try next provider.
                    break;
                }
            }
        }
        // كل المزودين فشلوا
        log.error({ tier, errors }, 'All providers failed — no fallback available');
        // Tag the final error with the most recent statusCode so callers can render
        // a tier-specific friendly message (429 → busy, 401 → config issue, etc.).
        const lastErr = errors[errors.length - 1];
        const lastStatus = lastErr && /\b(\d{3})\b/.exec(lastErr.error)?.[1]
            ? Number(/\b(\d{3})\b/.exec(lastErr.error)[1])
            : 503;
        throw new ModelError(`All providers failed for tier "${tier}": ${errors.map(e => `${e.provider}: ${e.error}`).join(' | ')}`, lastStatus, tier);
    }
    finally {
        // Restore the original model on the singleton provider, if we mutated it
        // for this call. Without this, a per-tenant choice would leak into the
        // next tenant's call (since providers are module singletons).
        if (restoreModel)
            restoreModel();
    }
}
/**
 * Look up the singleton ModelProvider instance for a provider name. Returns
 * null if the provider isn't configured (no API key) — caller should fall
 * back to the default chain.
 */
async function resolveProviderInstance(name) {
    switch (name) {
        case 'nvidia-nim':
        case 'nvidia':
            return nvidia;
        case 'anthropic':
            if (!process.env.ANTHROPIC_API_KEY)
                return null;
            return (await import('./anthropic.js')).anthropic;
        case 'openai':
            if (!process.env.OPENAI_API_KEY)
                return null;
            return (await import('./openai.js')).openai;
        case 'kimi':
            if (!process.env.MOONSHOT_API_KEY)
                return null;
            return (await import('./kimi.js')).kimi;
        case 'deepseek':
            if (!process.env.DEEPSEEK_API_KEY)
                return null;
            return (await import('./deepseek.js')).deepseek;
        case 'gemini':
            if (!process.env.GOOGLE_AI_API_KEY || process.env.GOOGLE_AI_API_KEY === 'mock_google_key')
                return null;
            return (await import('./gemini.js')).gemini;
        case 'ollama':
            return ollama; // Ollama probe-availability is checked inside the provider
        default:
            return null;
    }
}
// ─── Cost Calculator ─────────────────────────────────────────────────
import { MODELS } from '../config.js';
const COST_MAP = (() => {
    const m = {};
    for (const tier of [MODELS.router, MODELS.brain, MODELS.vision, MODELS.reasoning]) {
        m[tier.model] = { input: tier.costPer1MInput, output: tier.costPer1MOutput };
    }
    return m;
})();
/** حساب تكلفة الاستدعاء بالدولار */
export function calculateCost(model, tokensInput, tokensOutput) {
    const rates = COST_MAP[model];
    if (!rates)
        return 0;
    return (tokensInput / 1_000_000) * rates.input + (tokensOutput / 1_000_000) * rates.output;
}
//# sourceMappingURL=failover.js.map