{"_id":"@careob/llm-gateway","_rev":"3-5d03817b30b0abfd1895b78223015bd1","name":"@careob/llm-gateway","dist-tags":{"latest":"1.1.0"},"versions":{"1.0.0":{"name":"@careob/llm-gateway","version":"1.0.0","keywords":["llm","gateway","openai","anthropic","key-rotation","rate-limit","express"],"license":"MIT","_id":"@careob/llm-gateway@1.0.0","maintainers":[{"name":"abhishek_careob","email":"abhishek@careob.com"}],"dist":{"shasum":"d878518f57d3ee3567a3716415fcce9a9a2d6764","tarball":"https://registry.npmjs.org/@careob/llm-gateway/-/llm-gateway-1.0.0.tgz","fileCount":25,"integrity":"sha512-Ycc1vWdx97hlcNUkq2TmptRvbyXmPCzDFAQYq1VPBWZj/iE9ZdaAfESBNnlnF3QUnsBGH4b1NzwLx00/Td0Vuw==","signatures":[{"sig":"MEUCIHlgAXMobUChpT4uKkgbAmWZQQqR+68NDMPuWRVEDFFdAiEArFXKrvbZ+6O8K078/o6kbTCujUNm3wRCKM7NDWSLfSU=","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"unpackedSize":82270},"main":"dist/index.js","type":"module","types":"dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js"}},"scripts":{"dev":"tsc --watch","build":"tsc","typecheck":"tsc --noEmit"},"_npmUser":{"name":"abhishek_careob","email":"abhishek@careob.com"},"_npmVersion":"10.9.0","description":"Lightweight LLM gateway with multi-key rotation, rate-limit handling, provider failover, and spend tracking for Node.js/Express apps","directories":{},"_nodeVersion":"22.11.0","dependencies":{},"_hasShrinkwrap":false,"devDependencies":{"typescript":"^5.5.3","@types/node":"^20.14.9","@types/express":"^4.17.21"},"peerDependencies":{"express":"^4.18.0 || ^5.0.0"},"peerDependenciesMeta":{"express":{"optional":true}},"_npmOperationalInternal":{"tmp":"tmp/llm-gateway_1.0.0_1783082612948_0.331964910807661","host":"s3://npm-registry-packages-npm-production"}},"1.0.1":{"name":"@careob/llm-gateway","version":"1.0.1","keywords":["llm","gateway","openai","anthropic","key-rotation","rate-limit","express"],"author":{"name":"Careob","email":"gaurav@careob.com"},"license":"MIT","_id":"@careob/llm-gateway@1.0.1","maintainers":[{"name":"abhishek_careob","email":"abhishek@careob.com"}],"homepage":"https://github.com/careob/llm-gateway#readme","bugs":{"url":"https://github.com/careob/llm-gateway/issues"},"dist":{"shasum":"f1dac36fd7cbed5bba63dc0acddbda81f6a7e57f","tarball":"https://registry.npmjs.org/@careob/llm-gateway/-/llm-gateway-1.0.1.tgz","fileCount":26,"integrity":"sha512-W609azfur9e0nUFWxf5aNgxdC9qwxfHAMtXJ9zd/aLMRZ5FAUW9E4A6q/3QY+Qs/n18DZ4fUt2ozksADKzKG5w==","signatures":[{"sig":"MEYCIQDPQ0Gn8ZSQSAALuCDMevpmtGtwA1+pdvAor2mQAi+fqQIhALVV77OiTJdMXAPGrh+/tKbGsVhLMSPDC/9SOXDub29D","keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U"}],"attestations":{"url":"https://registry.npmjs.org/-/npm/v1/attestations/@careob%2fllm-gateway@1.0.1","provenance":{"predicateType":"https://slsa.dev/provenance/v1"}},"unpackedSize":87514},"main":"dist/index.js","type":"module","types":"dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js"}},"gitHead":"1bfdf41b8144cf4771c39fc07e1c40f3a832ebd1","scripts":{"dev":"tsc --watch","build":"tsc","typecheck":"tsc --noEmit"},"_npmUser":{"name":"abhishek_careob","email":"abhishek@careob.com"},"repository":{"url":"git+https://github.com/careob/llm-gateway.git","type":"git"},"_npmVersion":"10.9.8","description":"Lightweight LLM gateway with multi-key rotation, rate-limit handling, provider failover, and spend tracking for Node.js/Express apps","directories":{},"_nodeVersion":"22.23.1","dependencies":{},"_hasShrinkwrap":false,"devDependencies":{"typescript":"^5.5.3","@types/node":"^20.14.9","@types/express":"^4.17.21"},"peerDependencies":{"express":"^4.18.0 || ^5.0.0"},"peerDependenciesMeta":{"express":{"optional":true}},"_npmOperationalInternal":{"tmp":"tmp/llm-gateway_1.0.1_1783543234846_0.2761197258035877","host":"s3://npm-registry-packages-npm-production"}},"1.1.0":{"name":"@careob/llm-gateway","version":"1.1.0","description":"Lightweight LLM gateway with multi-key rotation, rate-limit handling, provider failover, and spend tracking for Node.js/Express apps","type":"module","main":"dist/index.js","types":"dist/index.d.ts","exports":{".":{"types":"./dist/index.d.ts","import":"./dist/index.js"}},"scripts":{"build":"tsc","typecheck":"tsc --noEmit","dev":"tsc --watch","test":"vitest run","test:watch":"vitest","test:coverage":"vitest run --coverage","ci":"npm run typecheck && npm run build && npm test"},"peerDependencies":{"express":"^4.18.0 || ^5.0.0"},"peerDependenciesMeta":{"express":{"optional":true}},"devDependencies":{"@types/express":"^4.17.21","@types/node":"^22.0.0","@vitest/coverage-v8":"^5.0.0","typescript":"^5.5.3","vite":"^8.3.0","vitest":"^5.0.0"},"keywords":["llm","gateway","openai","anthropic","key-rotation","rate-limit","express"],"license":"MIT","author":{"name":"Careob","email":"gaurav@careob.com"},"repository":{"type":"git","url":"git+https://github.com/careob/llm-gateway.git"},"homepage":"https://github.com/careob/llm-gateway#readme","bugs":{"url":"https://github.com/careob/llm-gateway/issues"},"_id":"@careob/llm-gateway@1.1.0","gitHead":"033ead4f03898c839cb9c9036feb7a6a83a3504e","_nodeVersion":"22.23.2","_npmVersion":"10.9.8","dist":{"integrity":"sha512-ZDYa+zBRhfZ5wC4p5ht5WazCJFk0qANnNugLDXmIAwwgSQ+ml6BayDRxFNvtAVRSkVUSHEra2JUiEj9Xv1N0aQ==","shasum":"29550b7f0d6f413b5c4b38a8b77be6948f3dc621","tarball":"https://registry.npmjs.org/@careob/llm-gateway/-/llm-gateway-1.1.0.tgz","fileCount":26,"unpackedSize":123895,"attestations":{"url":"https://registry.npmjs.org/-/npm/v1/attestations/@careob%2fllm-gateway@1.1.0","provenance":{"predicateType":"https://slsa.dev/provenance/v1"}},"signatures":[{"keyid":"SHA256:DhQ8wR5APBvFHLF/+Tc+AYvPOdTpcIDqOhxsBHRwC7U","sig":"MEYCIQDesdT4XAtGJo+NrYb9h9Dqi+xnCcflX/rfYm7sef8otgIhAOldjCH/Phg07PqA3RIy7Aa+QJhjSGJSnIixJErRu/Xc"}]},"_npmUser":{"name":"abhishek_careob","email":"abhishek@careob.com"},"directories":{},"maintainers":[{"name":"abhishek_careob","email":"abhishek@careob.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages-npm-production","tmp":"tmp/llm-gateway_1.1.0_1789260172737_0.03179291124399364"},"_hasShrinkwrap":false}},"time":{"created":"2026-07-03T12:43:32.767Z","modified":"2026-09-13T00:42:53.172Z","1.0.0":"2026-07-03T12:43:33.135Z","1.0.1":"2026-07-08T20:40:34.990Z","1.1.0":"2026-09-13T00:42:52.868Z"},"bugs":{"url":"https://github.com/careob/llm-gateway/issues"},"author":{"name":"Careob","email":"gaurav@careob.com"},"license":"MIT","homepage":"https://github.com/careob/llm-gateway#readme","keywords":["llm","gateway","openai","anthropic","key-rotation","rate-limit","express"],"repository":{"type":"git","url":"git+https://github.com/careob/llm-gateway.git"},"description":"Lightweight LLM gateway with multi-key rotation, rate-limit handling, provider failover, and spend tracking for Node.js/Express apps","maintainers":[{"name":"abhishek_careob","email":"abhishek@careob.com"}],"readme":"# @careob/llm-gateway\n\nLightweight LLM gateway for Node.js with multi-key rotation, rate-limit handling, provider failover, and spend tracking.\n\n## Features\n\n- **12+ providers** — OpenAI, Anthropic, Gemini, OpenRouter, DeepSeek, Groq, Mistral, Together, Qwen, Zhipu, Moonshot, Yi\n- **Key rotation** — round-robin across multiple API keys per provider\n- **Rate-limit handling** — automatic cooldown, exponential backoff with jitter on 429s\n- **Retry classification** — 400s fail fast, 401/403 quarantine keys, 429/5xx retry with backoff\n- **Provider failover** — capability-aware fallback to alternate providers\n- **Safe errors** — API keys and raw provider bodies are never exposed in error messages\n- **AbortSignal** — cancel requests with a caller-provided `AbortSignal`\n- **Spend tracking** — per-request cost estimation and usage logging\n- **Streaming** — SSE streaming with stream-idle timeout detection\n- **Express middleware** — drop-in proxy for Express apps\n- **Auto-discovery** — detects API keys from environment variables\n- **Zero dependencies** — only uses `express` as an optional peer dependency\n\n## Install\n\n```bash\nnpm install @careob/llm-gateway\n```\n\n## Quick Start\n\n```typescript\nimport { LLMGateway } from '@careob/llm-gateway';\n\nconst gateway = new LLMGateway({\n  keys: [\n    'sk-proj-your-openai-key',\n    { key: 'sk-ant-your-anthropic-key', provider: 'anthropic' },\n  ],\n  defaultModel: 'gpt-4o',\n});\n\nconst response = await gateway.chat({\n  messages: [{ role: 'user', content: 'Hello!' }],\n});\n\nconsole.log(response.choices[0].message.content);\n```\n\n### Auto-discover keys from environment\n\n```typescript\n// Reads OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY, etc.\nconst gateway = LLMGateway.fromEnv({ defaultModel: 'gpt-4o' });\n```\n\n## Multi-Key Rotation\n\nPass multiple keys for the same provider to distribute load:\n\n```typescript\nconst gateway = new LLMGateway({\n  keys: [\n    { key: 'sk-proj-key-1', provider: 'openai', label: 'prod-1' },\n    { key: 'sk-proj-key-2', provider: 'openai', label: 'prod-2' },\n    { key: 'sk-proj-key-3', provider: 'openai', label: 'prod-3' },\n  ],\n  defaultModel: 'gpt-4o',\n});\n```\n\nKeys are rotated round-robin. When a key hits a 429, it's automatically cooled down and the next key is used.\n\n## Streaming\n\n```typescript\nfor await (const chunk of gateway.chatStream({\n  model: 'gpt-4o',\n  messages: [{ role: 'user', content: 'Write a poem' }],\n})) {\n  process.stdout.write(chunk.choices[0]?.delta?.content ?? '');\n}\n```\n\n## OpenRouter\n\nAccess 200+ models through a single API key via [OpenRouter](https://openrouter.ai):\n\n```typescript\nconst gateway = new LLMGateway({\n  keys: ['sk-or-v1-your-openrouter-key'],\n  defaultModel: 'openai/gpt-4o',\n  openRouterReferer: 'https://your-app.com',\n  openRouterTitle: 'Your App Name',\n});\n\n// Use namespaced model identifiers\nconst response = await gateway.chat({\n  model: 'anthropic/claude-sonnet-4',\n  messages: [{ role: 'user', content: 'Hello!' }],\n});\n```\n\nOpenRouter keys (`sk-or-*`) are auto-detected. Models use namespaced identifiers like `openai/gpt-4o`, `anthropic/claude-sonnet-4`, `google/gemini-2.5-flash`, etc.\n\n## Express Proxy\n\nMount the gateway as an API endpoint in your Express app:\n\n```typescript\nimport express from 'express';\nimport { LLMGateway, createExpressProxy } from '@careob/llm-gateway';\n\nconst app = express();\napp.use(express.json());\n\nconst gateway = LLMGateway.fromEnv({ defaultModel: 'gpt-4o' });\nconst proxy = createExpressProxy(gateway, {\n  authorize: (req) => req.headers['x-api-key'] === 'your-secret',\n  extractMetadata: (req) => ({ userId: req.headers['x-user-id'] as string }),\n});\n\napp.post('/v1/chat/completions', proxy.chatCompletions);\napp.get('/v1/health', proxy.health);\n\napp.listen(3000);\n```\n\n## Spend Tracking\n\n```typescript\nconst gateway = new LLMGateway({\n  keys: ['sk-proj-...'],\n  defaultModel: 'gpt-4o',\n  onUsage: (log) => {\n    console.log(`${log.model} | ${log.totalTokens} tokens | $${log.estimatedCostUsd.toFixed(6)}`);\n    // Save to your database, send to analytics, etc.\n  },\n  onAllKeysExhausted: (provider) => {\n    console.warn(`All ${provider} keys exhausted!`);\n  },\n});\n```\n\n## Configuration\n\n| Option | Default | Description |\n|---|---|---|\n| `keys` | *required* | Array of API key strings or `KeyInput` objects |\n| `defaultModel` | — | Model to use when not specified in request |\n| `maxRetries` | `3` | Retries across keys before failing |\n| `cooldownMs` | `60000` | Cooldown duration for rate-limited keys (ms) |\n| `timeoutMs` | `30000` | Connection timeout (ms) |\n| `streamTimeoutMs` | `15000` | Stream idle timeout — no data received (ms) |\n| `onUsage` | — | Callback for usage/spend logging |\n| `onAllKeysExhausted` | — | Callback when all keys for a provider are exhausted |\n| `openRouterReferer` | — | Your site URL for OpenRouter rankings (sent as `HTTP-Referer`) |\n| `openRouterTitle` | — | Your app name for OpenRouter rankings (sent as `X-Title`) |\n\n### KeyInput\n\n```typescript\n{\n  key: string;\n  provider?: string;   // Auto-detected from key prefix if omitted\n  rpm?: number;        // Requests per minute limit\n  tpm?: number;        // Tokens per minute limit\n  label?: string;      // Human-readable label (e.g. \"prod-key-1\")\n}\n```\n\n## Supported Providers\n\n| Provider | Format | Models |\n|---|---|---|\n| OpenAI | openai | gpt-4o, gpt-4o-mini, o1, o3-mini, o4-mini, ... |\n| Anthropic | anthropic | claude-opus-4, claude-sonnet-4, claude-haiku-3.5 |\n| Google Gemini | gemini | gemini-2.5-pro, gemini-2.5-flash, ... |\n| OpenRouter | openai | openai/gpt-4o, anthropic/claude-sonnet-4, google/gemini-2.5-flash, ... |\n| DeepSeek | openai | deepseek-chat, deepseek-coder, deepseek-reasoner |\n| Groq | openai | llama-3.3-70b, mixtral-8x7b, ... |\n| Mistral | openai | mistral-large, mistral-small, codestral |\n| Together AI | openai | Llama-3.3-70B, DeepSeek-R1, ... |\n| Qwen | openai | qwen-turbo, qwen-plus, qwen-max |\n| Zhipu | openai | glm-4, glm-4-flash, glm-4-plus |\n| Moonshot | openai | moonshot-v1-8k/32k/128k |\n| Yi | openai | yi-large, yi-medium, yi-spark |\n\n## Health Check\n\n```typescript\nconst stats = gateway.getStats();\n// [{ provider: 'openai', label: 'prod-1', healthy: true, requestsThisMinute: 12, ... }]\n```\n\n## Development\n\n```bash\nnpm test              # run tests\nnpm run test:coverage # run tests with coverage report\nnpm run ci            # typecheck + build + test\n```\n\n## License\n\nMIT\n","readmeFilename":"README.md"}