# Provider abstraction — one interface, every model

The full normalization layer. Once these adapters exist, swapping OpenAI ↔ Anthropic
↔ Gemini ↔ OSS is a config string. The agent loop, the RAG embedder, the LLM-as-judge,
and the cost router all consume the *same* `LLMProvider`. Async-first; Python 3.12+,
Pydantic v2.

## The interface

```python
from __future__ import annotations

from typing import AsyncIterator, Literal, Protocol, runtime_checkable

from pydantic import BaseModel, Field


class Message(BaseModel):
    role: Literal["system", "user", "assistant", "tool"]
    content: str
    tool_call_id: str | None = None  # set when role == "tool"
    name: str | None = None          # tool name for tool-result turns


class ToolSpec(BaseModel):
    name: str
    description: str
    parameters: dict  # JSON Schema (object) describing the arguments


class ToolCall(BaseModel):
    id: str
    name: str
    arguments: dict = Field(default_factory=dict)


class Usage(BaseModel):
    input_tokens: int = 0
    output_tokens: int = 0
    cached_input_tokens: int = 0
    cost_usd: float = 0.0


class CompletionRequest(BaseModel):
    model: str
    messages: list[Message]
    tools: list[ToolSpec] = Field(default_factory=list)
    response_schema: dict | None = None
    temperature: float = 0.0
    top_p: float | None = None
    max_tokens: int = 1024
    stop: list[str] | None = None
    cache_system: bool = True  # opt-in prompt caching for the system block


class CompletionResponse(BaseModel):
    text: str = ""
    tool_calls: list[ToolCall] = Field(default_factory=list)
    finish_reason: Literal["stop", "tool_calls", "length", "content_filter"] = "stop"
    usage: Usage = Field(default_factory=Usage)
    raw: dict | None = None


class StreamEvent(BaseModel):
    type: Literal["text", "tool_call", "usage", "done"]
    text: str = ""
    tool_call: ToolCall | None = None
    usage: Usage | None = None


@runtime_checkable
class LLMProvider(Protocol):
    async def complete(self, req: CompletionRequest) -> CompletionResponse: ...
    async def stream(self, req: CompletionRequest) -> AsyncIterator[StreamEvent]: ...
    async def embed(self, texts: list[str]) -> list[list[float]]: ...
```

## Four adapters (full code)

### OpenAI — Responses API primary

Prefer the **Responses API** for agent/tool loops; the Chat Completions shape still
works (uncomment the fallback). Model ids resolve from config — never literals here.

```python
from openai import AsyncOpenAI


class OpenAIAdapter:
    def __init__(self, model: str, embed_model: str = "text-embedding-3-small") -> None:
        self.model, self.embed_model, self.client = model, embed_model, AsyncOpenAI()

    async def complete(self, req: CompletionRequest) -> CompletionResponse:
        kwargs: dict = {
            "model": self.model,
            "input": [{"role": m.role, "content": m.content} for m in req.messages],
            "temperature": req.temperature,
            "max_output_tokens": req.max_tokens,
        }
        if req.tools:
            kwargs["tools"] = [
                {"type": "function", "name": t.name, "description": t.description, "parameters": t.parameters}
                for t in req.tools
            ]
        if req.response_schema:
            kwargs["text"] = {"format": {"type": "json_schema", "name": "out",
                                         "schema": req.response_schema, "strict": True}}
        r = await self.client.responses.create(**kwargs)
        # Fallback (Chat Completions): r = await self.client.chat.completions.create(
        #     model=self.model, messages=[m.model_dump(exclude_none=True) for m in req.messages], ...)
        calls = [
            ToolCall(id=item.call_id, name=item.name, arguments=_loads(item.arguments))
            for item in r.output if item.type == "function_call"
        ]
        return CompletionResponse(
            text=r.output_text or "",
            tool_calls=calls,
            finish_reason="tool_calls" if calls else "stop",
            usage=Usage(input_tokens=r.usage.input_tokens, output_tokens=r.usage.output_tokens,
                        cached_input_tokens=getattr(r.usage.input_tokens_details, "cached_tokens", 0)),
            raw=r.model_dump(),
        )

    async def embed(self, texts: list[str]) -> list[list[float]]:
        r = await self.client.embeddings.create(model=self.embed_model, input=texts)
        return [d.embedding for d in r.data]
```

### Anthropic — `input_schema`, top-level `system`, `cache_control`

```python
from anthropic import AsyncAnthropic


class AnthropicAdapter:
    def __init__(self, model: str) -> None:
        self.model, self.client = model, AsyncAnthropic()

    async def complete(self, req: CompletionRequest) -> CompletionResponse:
        sys_text = "\n".join(m.content for m in req.messages if m.role == "system")
        system = None
        if sys_text:  # cache the (large, stable) system block when asked
            block = {"type": "text", "text": sys_text}
            if req.cache_system:
                block["cache_control"] = {"type": "ephemeral"}
            system = [block]
        turns = [_anthropic_turn(m) for m in req.messages if m.role != "system"]
        kwargs: dict = {"model": self.model, "system": system, "messages": turns,
                        "max_tokens": req.max_tokens, "temperature": req.temperature}
        if req.tools:
            kwargs["tools"] = [{"name": t.name, "description": t.description, "input_schema": t.parameters}
                               for t in req.tools]
        if req.response_schema:  # structured output via tool-forcing
            kwargs["tools"] = [{"name": "out", "description": "Emit the result", "input_schema": req.response_schema}]
            kwargs["tool_choice"] = {"type": "tool", "name": "out"}
        r = await self.client.messages.create(**kwargs)
        text = "".join(b.text for b in r.content if b.type == "text")
        calls = [ToolCall(id=b.id, name=b.name, arguments=b.input) for b in r.content if b.type == "tool_use"]
        if req.response_schema and calls:  # tool-forced JSON -> surface as text
            text = _dumps(calls[0].arguments)
            calls = []
        return CompletionResponse(
            text=text, tool_calls=calls,
            finish_reason="tool_calls" if calls else "stop",
            usage=Usage(input_tokens=r.usage.input_tokens, output_tokens=r.usage.output_tokens,
                        cached_input_tokens=getattr(r.usage, "cache_read_input_tokens", 0)),
            raw=r.model_dump(),
        )

    async def embed(self, texts: list[str]) -> list[list[float]]:
        raise NotImplementedError("Anthropic has no embeddings API; use an embed-only provider")
```

### Gemini — `responseSchema` + `responseMimeType`

```python
from google import genai
from google.genai import types


class GeminiAdapter:
    def __init__(self, model: str, embed_model: str = "gemini-embedding-001") -> None:
        self.model, self.embed_model, self.client = model, embed_model, genai.Client()

    async def complete(self, req: CompletionRequest) -> CompletionResponse:
        sys_text = "\n".join(m.content for m in req.messages if m.role == "system") or None
        contents = [
            types.Content(role="model" if m.role == "assistant" else "user",
                          parts=[types.Part(text=m.content)])
            for m in req.messages if m.role in ("user", "assistant")
        ]
        cfg = types.GenerateContentConfig(
            system_instruction=sys_text, temperature=req.temperature, max_output_tokens=req.max_tokens,
        )
        if req.tools:
            # Raw JSON Schema dicts go in *_json_schema (current SDK). The legacy
            # `parameters=`/`response_schema=` fields expect a proto Schema / Pydantic
            # type and are mutually exclusive with these — don't mix them.
            cfg.tools = [types.Tool(function_declarations=[
                types.FunctionDeclaration(name=t.name, description=t.description,
                                          parameters_json_schema=t.parameters)
                for t in req.tools])]
        if req.response_schema:
            cfg.response_mime_type = "application/json"
            cfg.response_json_schema = req.response_schema
        r = await self.client.aio.models.generate_content(model=self.model, contents=contents, config=cfg)
        calls = [ToolCall(id=f"call_{i}", name=fc.name, arguments=dict(fc.args))
                 for i, fc in enumerate(r.function_calls or [])]
        um = r.usage_metadata
        return CompletionResponse(
            text=r.text or "", tool_calls=calls,
            finish_reason="tool_calls" if calls else "stop",
            usage=Usage(input_tokens=um.prompt_token_count, output_tokens=um.candidates_token_count or 0,
                        cached_input_tokens=um.cached_content_token_count or 0),
            raw=r.model_dump(),
        )

    async def embed(self, texts: list[str]) -> list[list[float]]:
        r = await self.client.aio.models.embed_content(model=self.embed_model, contents=texts)
        return [e.values for e in r.embeddings]
```

### OSS via litellm — one OpenAI-format call to any `base_url`/model

```python
import litellm  # pin v1.85.x (as of 2026-06; avoid quarantined 1.82.7/1.82.8)


class LiteLLMAdapter:
    """vLLM / Ollama / TGI / 100+ hosted models behind one OpenAI-format surface."""

    def __init__(self, model: str, base_url: str | None = None, embed_model: str | None = None) -> None:
        self.model, self.base_url, self.embed_model = model, base_url, embed_model

    async def complete(self, req: CompletionRequest) -> CompletionResponse:
        kwargs: dict = {
            "model": self.model, "api_base": self.base_url,
            "messages": [m.model_dump(exclude_none=True) for m in req.messages],
            "temperature": req.temperature, "max_tokens": req.max_tokens,
        }
        if req.tools:
            kwargs["tools"] = to_openai_tools(req.tools)
        if req.response_schema:
            kwargs["response_format"] = {"type": "json_schema",
                "json_schema": {"name": "out", "schema": req.response_schema, "strict": True}}
        r = await litellm.acompletion(**kwargs)
        msg = r.choices[0].message
        calls = [ToolCall(id=c.id, name=c.function.name, arguments=_loads(c.function.arguments))
                 for c in (msg.tool_calls or [])]
        return CompletionResponse(
            text=msg.content or "", tool_calls=calls,
            finish_reason="tool_calls" if calls else "stop",
            usage=Usage(input_tokens=r.usage.prompt_tokens, output_tokens=r.usage.completion_tokens,
                        cost_usd=litellm.completion_cost(r) or 0.0),
            raw=r.model_dump(),
        )

    async def embed(self, texts: list[str]) -> list[list[float]]:
        r = await litellm.aembedding(model=self.embed_model or self.model, input=texts, api_base=self.base_url)
        return [d["embedding"] for d in r.data]
```

```python
import json


def _loads(s: str | dict) -> dict:
    return s if isinstance(s, dict) else json.loads(s or "{}")


def _dumps(d: dict) -> str:
    return json.dumps(d, ensure_ascii=False)


def _anthropic_turn(m: Message) -> dict:
    if m.role == "tool":  # tool result -> a user turn carrying tool_result content
        return {"role": "user", "content": [{"type": "tool_result", "tool_use_id": m.tool_call_id, "content": m.content}]}
    return {"role": m.role, "content": m.content}
```

## Quirk matrix (as of 2026-06; verify before quoting)

| Aspect | OpenAI | Anthropic | Gemini | OSS / litellm |
|---|---|---|---|---|
| Tool schema | `tools[].function.parameters` (JSON Schema) | `tools[].input_schema` | `FunctionDeclaration.parameters_json_schema` | OpenAI shape |
| Structured output | strict JSON Schema (`text.format`) | tool-forcing (`tool_choice`) | `response_json_schema` + `response_mime_type` | strict JSON Schema if backend supports |
| System prompt | `system` role message | top-level `system=` param | `system_instruction` in config | `system` role message |
| Streaming event | `response.output_text.delta` | `content_block_delta` | chunk stream (`.text` per chunk) | OpenAI delta |
| Prompt caching | automatic cached input (−90% cached) | explicit `cache_control` (−90% read) | implicit/explicit cached content | backend-dependent |
| Max context | model-dependent; >272K billed 2×/1.5× | 1M flat-rate on Opus/Sonnet | 1M on Flash | backend-dependent |
| JSON-mode strictness | strict (validated) | only as strict as the tool schema | enforced by `responseSchema` | backend-dependent |

## Structured / JSON output

One entrypoint that picks the right mechanism per provider and validates, with a
bounded retry that feeds the validation error back to the model.

```python
from pydantic import BaseModel, ValidationError


async def complete_structured(provider: LLMProvider, req: CompletionRequest,
                              schema: type[BaseModel], max_retries: int = 2) -> BaseModel:
    req = req.model_copy(update={"response_schema": schema.model_json_schema(), "temperature": 0.0})
    last_err: str = ""
    for attempt in range(max_retries + 1):
        if attempt:  # feed the prior validation error back as a correction turn
            req.messages.append(Message(role="user",
                content=f"Your previous output failed validation: {last_err}. Re-emit valid JSON only."))
        resp = await provider.complete(req)
        try:
            return schema.model_validate_json(resp.text)
        except ValidationError as e:
            last_err = str(e)
    raise ValueError(f"structured output failed after {max_retries} retries: {last_err}")
```

## Tool / function-calling normalization

Convert one `list[ToolSpec]` into each wire format, parse tool calls back into the
normalized `ToolCall`, and build each provider's tool-result turn.

```python
def to_openai_tools(tools: list[ToolSpec]) -> list[dict]:
    return [{"type": "function", "function": {"name": t.name, "description": t.description,
            "parameters": t.parameters}} for t in tools]


def to_anthropic_tools(tools: list[ToolSpec]) -> list[dict]:
    return [{"name": t.name, "description": t.description, "input_schema": t.parameters} for t in tools]


def to_gemini_tools(tools: list[ToolSpec]) -> list[dict]:
    # parameters_json_schema (not parameters) for raw JSON Schema dicts.
    return [{"function_declarations": [{"name": t.name, "description": t.description,
            "parameters_json_schema": t.parameters} for t in tools]}]


def parse_tool_calls(provider: str, raw: dict) -> list[ToolCall]:
    if provider == "anthropic":
        return [ToolCall(id=b["id"], name=b["name"], arguments=b["input"])
                for b in raw.get("content", []) if b.get("type") == "tool_use"]
    if provider == "gemini":
        fcs = raw.get("candidates", [{}])[0].get("content", {}).get("parts", [])
        return [ToolCall(id=f"call_{i}", name=p["functionCall"]["name"], arguments=p["functionCall"].get("args", {}))
                for i, p in enumerate(fcs) if "functionCall" in p]
    # openai / litellm
    msg = raw["choices"][0]["message"]
    return [ToolCall(id=c["id"], name=c["function"]["name"], arguments=_loads(c["function"]["arguments"]))
            for c in (msg.get("tool_calls") or [])]


def tool_result_message(provider: str, call: ToolCall, result: str) -> Message:
    if provider == "openai":  # OpenAI/litellm: a "tool" role turn keyed by call id
        return Message(role="tool", tool_call_id=call.id, name=call.name, content=result)
    if provider == "anthropic":  # carried inside a user turn (see _anthropic_turn)
        return Message(role="tool", tool_call_id=call.id, name=call.name, content=result)
    return Message(role="user", name=call.name, content=result)  # gemini: functionResponse part
```

## Streaming

One normalized async generator. Each provider's native event maps onto `StreamEvent`.

```python
async def stream_openai(client, model: str, req: CompletionRequest) -> AsyncIterator[StreamEvent]:
    async with client.responses.stream(model=model,
            input=[{"role": m.role, "content": m.content} for m in req.messages]) as s:
        async for event in s:
            if event.type == "response.output_text.delta":      # OpenAI delta
                yield StreamEvent(type="text", text=event.delta)
            elif event.type == "response.function_call_arguments.done":
                yield StreamEvent(type="tool_call",
                    tool_call=ToolCall(id=event.item_id, name=event.name, arguments=_loads(event.arguments)))
        final = await s.get_final_response()
        yield StreamEvent(type="usage", usage=Usage(input_tokens=final.usage.input_tokens,
                                                     output_tokens=final.usage.output_tokens))
    yield StreamEvent(type="done")
# Anthropic maps `content_block_delta` -> text and `content_block_start(tool_use)` -> tool_call.
# Gemini yields chunks; emit StreamEvent(type="text", text=chunk.text) per chunk, usage at the end.
```

## Token & context-window management

```python
import tiktoken


def count_tokens(provider: str, model: str, messages: list[Message]) -> int:
    text = "\n".join(m.content for m in messages)
    if provider in ("openai", "litellm"):
        try:
            enc = tiktoken.encoding_for_model(model)
        except KeyError:
            enc = tiktoken.get_encoding("o200k_base")
        return len(enc.encode(text)) + 4 * len(messages)  # per-message overhead
    # Anthropic/Gemini expose count_tokens server-side; fall back to a heuristic offline.
    return len(text) // 4 + 4 * len(messages)  # chars/4 approximation


def trim_to_budget(messages: list[Message], provider: str, model: str, max_tokens: int) -> list[Message]:
    system = [m for m in messages if m.role == "system"]      # invariant: never dropped
    body = [m for m in messages if m.role != "system"]
    while body and count_tokens(provider, model, system + body) > max_tokens:
        body.pop(0)  # drop oldest turn first; replace with a summary turn if you have one
    return system + body


def preflight(messages: list[Message], provider: str, model: str, ctx_limit: int, reserve_out: int) -> None:
    used = count_tokens(provider, model, messages)
    if used + reserve_out > ctx_limit:
        raise ValueError(f"prompt {used} + reserve {reserve_out} exceeds context {ctx_limit}; trim first")
```

## Sampling params

```python
def normalize_sampling(req: CompletionRequest) -> dict:
    out: dict = {"temperature": req.temperature, "max_tokens": req.max_tokens}
    if req.top_p is not None:
        out["top_p"] = req.top_p
    if req.stop:
        out["stop"] = req.stop
    return out
```

| Param | OpenAI | Anthropic | Gemini |
|---|---|---|---|
| `max_tokens` | `max_output_tokens` (Responses) | `max_tokens` (required) | `max_output_tokens` |
| `stop` | `stop` | `stop_sequences` | `stop_sequences` |
| `temperature` | 0–2 | 0–1 | 0–2 |

Note (as of 2026-06; verify before quoting): some reasoning models ignore
`temperature`/`top_p`. Treat sampling as advisory, not guaranteed.

## Config-driven selection

```python
import os
from functools import lru_cache

# Roles, not raw ids. App code says route("cheap"); ids live here and rot here only.
MODEL_REGISTRY: dict[str, str] = {
    "default": "anthropic:claude-sonnet-4-6",
    "cheap":   "anthropic:claude-haiku-4-5",
    "smart":   "anthropic:claude-opus-4-8",   # Opus 4.8 (current as of 2026-06); ids rot — re-verify
}


@lru_cache(maxsize=None)
def get_provider(spec: str) -> LLMProvider:
    provider, _, model = spec.partition(":")
    return {"openai": OpenAIAdapter, "anthropic": AnthropicAdapter,
            "gemini": GeminiAdapter, "litellm": LiteLLMAdapter}[provider](model)


def route(role: str = "default") -> tuple[LLMProvider, str]:
    spec = os.environ.get(f"LLM_{role.upper()}") or MODEL_REGISTRY[role]  # env overrides registry
    return get_provider(spec), spec.split(":", 1)[1]
```

## litellm note (as of 2026-06)

Adopt **litellm** (or its proxy) as the adapter when you must reach many providers/OSS
endpoints, want Router-level fallbacks and per-key budgets, or run a central LLM gateway.
Hand-roll the four adapters above when you only target one or two providers and want zero
extra deps. Pin `v1.85.x`; avoid the quarantined `1.82.7`/`1.82.8` (2026-03 supply-chain
advisory). Router fallbacks/budgets:

```python
from litellm import Router

router = Router(
    model_list=[
        {"model_name": "default",
         "litellm_params": {"model": "anthropic/claude-sonnet-4-6"}},
        {"model_name": "default",  # same alias = automatic fallback target
         "litellm_params": {"model": "openai/gpt-5.4"}},
    ],
    fallbacks=[{"default": ["default"]}],
    routing_strategy="latency-based-routing",
)
resp = await router.acompletion(model="default", messages=[{"role": "user", "content": "hi"}])
```

## See also

- `agent-loops-and-harness.md` — the loop that drives `provider.complete`/`stream`.
- `evals-and-observability.md` — the cost router and LLM-as-judge use this same interface.
