{"_id":"@bunnyblue/realtime-api-beta","_rev":"4-12725328ca8cafb94190c058f2394e6f","name":"@bunnyblue/realtime-api-beta","dist-tags":{"latest":"1.1071.4"},"versions":{"1.0.0":{"name":"@bunnyblue/realtime-api-beta","version":"1.0.0","author":{"name":"Keith Horwood","email":"khorwood@openai.com"},"license":"MIT","_id":"@bunnyblue/realtime-api-beta@1.0.0","maintainers":[{"name":"bunnyblue","email":"bunnyblueair@gmail.com"}],"dist":{"shasum":"5b3bc45ab696006e04ac5e39c32fc83633ebc908","tarball":"https://registry.npmjs.org/@bunnyblue/realtime-api-beta/-/realtime-api-beta-1.0.0.tgz","fileCount":11,"integrity":"sha512-g5LIhe96bKPtYSj+3Xuo2G4RLqhgO9lcV9e2tYSp7fPN4e9kPZfLmXkCUJXr+wEGYRqFKSv3qOg1gfjcenqw5A==","signatures":[{"sig":"MEUCIQDqpSfrxBdWWB6Oa6XeD2U26lo3WZlVDPkQYwjftxubAgIgVT7OLM6lZwD5d2G/+QsLYZ89nV1SdymmETlSxQkWtqY=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":59063},"main":"index.js","type":"module","types":"./dist/index.d.ts","gitHead":"9812e4510d18eb680e4d35920c76c008ae5f6fc8","private":false,"scripts":{"test":"mocha ./test/index.js"},"_npmUser":{"name":"bunnyblue","email":"bunnyblueair@gmail.com"},"_npmVersion":"10.8.2","description":"OpenAI Realtime API Reference Client (beta)","directories":{},"_nodeVersion":"20.17.0","dependencies":{"ws":"^8.18.0"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"chai":"^5.1.1","mocha":"^10.7.3","dotenv":"^16.4.5","prettier":"^3.3.3","websocket":"^1.0.35","typescript":"^5.6.2","audio-decode":"^2.2.2"},"_npmOperationalInternal":{"tmp":"tmp/realtime-api-beta_1.0.0_1729152475553_0.836806884749516","host":"s3://npm-registry-packages"}},"1.1071.1":{"name":"@bunnyblue/realtime-api-beta","version":"1.1071.1","author":{"name":"Keith Horwood","email":"khorwood@openai.com"},"license":"MIT","_id":"@bunnyblue/realtime-api-beta@1.1071.1","maintainers":[{"name":"bunnyblue","email":"bunnyblueair@gmail.com"}],"dist":{"shasum":"5da6437f23e609b1d5cef0cef4dd6d9224d75c52","tarball":"https://registry.npmjs.org/@bunnyblue/realtime-api-beta/-/realtime-api-beta-1.1071.1.tgz","fileCount":23,"integrity":"sha512-0ZppB/yLqQbRzr9BZE5QVH2CL1jbxH/VIUS7t0Dp2EF1VyMNzrV27K7wTNWAfW4akS9I82ru+hnkJa9Ro/fblQ==","signatures":[{"sig":"MEYCIQDuTUBL121aTANYS6Jwz4edwtJcE8FGkPO9TiV5wkFMRgIhAMQ1z97VHHk9uA4UoG7hl/9m70Hf5VMIoNSVVDfgckJb","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":88333},"main":"index.js","type":"module","types":"./dist/index.d.ts","gitHead":"c1ee4b6344ed4293c30531039a99f25ef6a9bd48","private":false,"scripts":{"test":"mocha ./test/index.js","build":"tsc"},"_npmUser":{"name":"bunnyblue","email":"bunnyblueair@gmail.com"},"_npmVersion":"10.8.2","description":"OpenAI Realtime API Reference Client (beta)","directories":{},"_nodeVersion":"20.17.0","dependencies":{"ws":"^8.18.0"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"chai":"^5.1.1","mocha":"^10.7.3","dotenv":"^16.4.5","prettier":"^3.3.3","websocket":"^1.0.35","typescript":"^5.6.3","audio-decode":"^2.2.2"},"_npmOperationalInternal":{"tmp":"tmp/realtime-api-beta_1.1071.1_1729153275220_0.4910444339351605","host":"s3://npm-registry-packages"}},"1.1071.2":{"name":"@bunnyblue/realtime-api-beta","version":"1.1071.2","author":{"name":"Keith Horwood","email":"khorwood@openai.com"},"license":"MIT","_id":"@bunnyblue/realtime-api-beta@1.1071.2","maintainers":[{"name":"bunnyblue","email":"bunnyblueair@gmail.com"}],"dist":{"shasum":"ea33d1aabb79c0984959f9cb726dc1d13dcca9ab","tarball":"https://registry.npmjs.org/@bunnyblue/realtime-api-beta/-/realtime-api-beta-1.1071.2.tgz","fileCount":16,"integrity":"sha512-FbhuIrIkCgYsjsDCu/NQy3/mgAdEslg3dZOSOHeCt7kS+BTelUzNYdfBdMnMI+rydXHgTSfpY+2PiuuDPCvF+w==","signatures":[{"sig":"MEUCIQDOG/C2tKXMIYcF/7Hgy7uoxc44kjvicuHS3OzaQTadUAIgGd6wsL8bcx0e+A6SuEWXeq9o92ltUCDpS8yFp87CfxM=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":44594},"main":"index.js","type":"module","types":"./dist/index.d.ts","exports":{".":{"import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"3754bd4d6ed1249d7e6e37fbde4a362247dd7a25","private":false,"scripts":{"test":"mocha ./test/index.js","build":"tsc"},"_npmUser":{"name":"bunnyblue","email":"bunnyblueair@gmail.com"},"_npmVersion":"10.8.2","description":"OpenAI Realtime API Reference Client (beta)","directories":{},"_nodeVersion":"20.17.0","dependencies":{"ws":"^8.18.0"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"chai":"^5.1.1","mocha":"^10.7.3","dotenv":"^16.4.5","prettier":"^3.3.3","websocket":"^1.0.35","typescript":"^5.6.3","audio-decode":"^2.2.2"},"_npmOperationalInternal":{"tmp":"tmp/realtime-api-beta_1.1071.2_1729155727520_0.09037605168500984","host":"s3://npm-registry-packages"}},"1.1071.3":{"name":"@bunnyblue/realtime-api-beta","version":"1.1071.3","author":{"name":"Keith Horwood","email":"khorwood@openai.com"},"license":"MIT","_id":"@bunnyblue/realtime-api-beta@1.1071.3","maintainers":[{"name":"bunnyblue","email":"bunnyblueair@gmail.com"}],"dist":{"shasum":"38a9aba2da4cf3f6c69fdd1f5e1373e2e52afd3d","tarball":"https://registry.npmjs.org/@bunnyblue/realtime-api-beta/-/realtime-api-beta-1.1071.3.tgz","fileCount":21,"integrity":"sha512-iiLzPpknYA79lCYpejjguFT+jwuHV+RMWbB6BsAUM+8sNJhAvGjaszg8NmJKsFsEZvR1pzOGskemxVWk2O3oLA==","signatures":[{"sig":"MEUCIAtaeXhp/TXIwnGdW8BmaTIYIeI/I0E/whYG++XeeuYyAiEAhxgTBZQtd3GtPVMbVTpDjtzR8TCZTdOt0XqC/0TMTBM=","keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA"}],"unpackedSize":88210},"main":"index.js","type":"module","types":"./dist/index.d.ts","exports":{".":{"import":"./dist/index.js","require":"./dist/index.cjs"}},"gitHead":"e0189b5353b68928707fcbc6c785f789c2f067a0","private":false,"scripts":{"test":"mocha ./test/index.js","build":"tsc"},"_npmUser":{"name":"bunnyblue","email":"bunnyblueair@gmail.com"},"_npmVersion":"10.8.2","description":"OpenAI Realtime API Reference Client (beta)","directories":{},"_nodeVersion":"20.17.0","dependencies":{"ws":"^8.18.0"},"publishConfig":{"access":"public"},"_hasShrinkwrap":false,"devDependencies":{"chai":"^5.1.1","mocha":"^10.7.3","dotenv":"^16.4.5","prettier":"^3.3.3","websocket":"^1.0.35","typescript":"^5.6.3","audio-decode":"^2.2.2"},"_npmOperationalInternal":{"tmp":"tmp/realtime-api-beta_1.1071.3_1729156044260_0.3259773232138288","host":"s3://npm-registry-packages"}},"1.1071.4":{"name":"@bunnyblue/realtime-api-beta","version":"1.1071.4","private":false,"publishConfig":{"access":"public"},"type":"module","description":"OpenAI Realtime API Reference Client (beta)","main":"index.js","types":"./dist/index.d.ts","scripts":{"build":"tsc","test":"mocha ./test/index.js"},"author":{"name":"Keith Horwood","email":"khorwood@openai.com"},"license":"MIT","devDependencies":{"audio-decode":"^2.2.2","chai":"^5.1.1","dotenv":"^16.4.5","mocha":"^10.7.3","prettier":"^3.3.3","typescript":"^5.6.3","websocket":"^1.0.35"},"dependencies":{"ws":"^8.18.0"},"_id":"@bunnyblue/realtime-api-beta@1.1071.4","gitHead":"c60bf45cacc3b45b5858f8804928259bb55db300","_nodeVersion":"20.17.0","_npmVersion":"10.8.2","dist":{"integrity":"sha512-YAAvmq//YgIzOocwjwqQw78oyY0K5Cc42AOR+L0XPxpB8AVQXRTCNRRBz74VWLTJ4QSM0N7TdtVmqDyc3P6Urw==","shasum":"d288d7f7f3053baa887220cebd854e08809a3724","tarball":"https://registry.npmjs.org/@bunnyblue/realtime-api-beta/-/realtime-api-beta-1.1071.4.tgz","fileCount":21,"unpackedSize":88103,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQDMtJCV5qAt2JjuBlaVLFsz7+O+G7dHryOG97Zhym3eVgIgftKqIRS23+mYt5wCVdvrX42oqWmnjbin7HaDTOaZ51Y="}]},"_npmUser":{"name":"bunnyblue","email":"bunnyblueair@gmail.com"},"directories":{},"maintainers":[{"name":"bunnyblue","email":"bunnyblueair@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/realtime-api-beta_1.1071.4_1729157493548_0.5569460626987157"},"_hasShrinkwrap":false}},"time":{"created":"2024-10-17T08:07:55.434Z","modified":"2024-10-17T09:31:33.884Z","1.0.0":"2024-10-17T08:07:55.712Z","1.1071.1":"2024-10-17T08:21:15.432Z","1.1071.2":"2024-10-17T09:02:07.760Z","1.1071.3":"2024-10-17T09:07:24.421Z","1.1071.4":"2024-10-17T09:31:33.728Z"},"author":{"name":"Keith Horwood","email":"khorwood@openai.com"},"license":"MIT","description":"OpenAI Realtime API Reference Client (beta)","maintainers":[{"name":"bunnyblue","email":"bunnyblueair@gmail.com"}],"readme":"# Reference Client: Realtime API (beta)\n\nThis repository contains a reference client aka sample library for connecting\nto OpenAI's Realtime API.\n**This library is in beta and should not be treated as a final implementation.**\nYou can use it to easily prototype conversational apps.\n\n**The easiest way to get playing with the API right away** is to use the\n[**Realtime Console**](https://github.com/openai/openai-realtime-console), it uses\nthe reference client to deliver a fully-functional API inspector with examples\nof voice visualization and more.\n\n# Quickstart\n\nThis library is built to be used both server-side (Node.js) and in browser (React, Vue),\nin both JavaScript and TypeScript codebases. While in beta, to install the library you will\nneed to `npm install` directly from the GitHub repository.\n\n```shell\n$ npm i openai/openai-realtime-api-beta --save\n```\n\n```javascript\nimport { RealtimeClient } from '@openai/realtime-api-beta';\n\nconst client = new RealtimeClient({ apiKey: process.env.OPENAI_API_KEY });\n\n// Can set parameters ahead of connecting, either separately or all at once\nclient.updateSession({ instructions: 'You are a great, upbeat friend.' });\nclient.updateSession({ voice: 'alloy' });\nclient.updateSession({\n  turn_detection: { type: 'none' }, // or 'server_vad'\n  input_audio_transcription: { model: 'whisper-1' },\n});\n\n// Set up event handling\nclient.on('conversation.updated', (event) => {\n  const { item, delta } = event;\n  const items = client.conversation.getItems();\n  /**\n   * item is the current item being updated\n   * delta can be null or populated\n   * you can fetch a full list of items at any time\n   */\n});\n\n// Connect to Realtime API\nawait client.connect();\n\n// Send a item and triggers a generation\nclient.sendUserMessageContent([{ type: 'input_text', text: `How are you?` }]);\n```\n\n## Browser (front-end) quickstart\n\nYou can use this client directly from the browser in e.g. React or Vue apps.\n**We do not recommend this, your API keys are at risk if you connect to OpenAI directly from the browser.**\nIn order to instantiate the client in a browser environment, use:\n\n```javascript\nimport { RealtimeClient } from '@openai/realtime-api-beta';\n\nconst client = new RealtimeClient({\n  apiKey: process.env.OPENAI_API_KEY,\n  dangerouslyAllowAPIKeyInBrowser: true,\n});\n```\n\nIf you are running your own relay server, e.g. with the\n[Realtime Console](https://github.com/openai/openai-realtime-console), you can\ninstead connect to the relay server URL like so:\n\n```javascript\nconst client = new RealtimeClient({ url: RELAY_SERVER_URL });\n```\n\n# Table of contents\n\n1. [Project structure](#project-structure)\n1. [Using the reference client](#using-the-reference-client)\n   1. [Sending messages](#sending-messages)\n   1. [Sending streaming audio](#sending-streaming-audio)\n   1. [Adding and using tools](#adding-and-using-tools)\n      1. [Manually using tools](#manually-using-tools)\n   1. [Interrupting the model](#interrupting-the-model)\n1. [Client events](#client-events)\n   1. [Reference client utility events](#reference-client-utility-events)\n1. [Server events](#server-events)\n1. [Running tests](#running-tests)\n1. [Acknowledgements and contact](#acknowledgements-and-contact)\n\n# Project structure\n\nIn this library, there are three primitives for interfacing with the Realtime API.\nWe recommend starting with the `RealtimeClient`, but more advanced users may be\nmore comfortable working closer to the metal.\n\n1. [`RealtimeClient`](./lib/client.js)\n   - Primary abstraction for interfacing with the Realtime API\n   - Enables rapid application development with a simplified control flow\n   - Has custom `conversation.updated`, `conversation.item.appended`, `conversation.item.completed`, `conversation.interrupted` and `realtime.event` events\n   - These events send item deltas and conversation history\n1. [`RealtimeAPI`](./lib/api.js)\n   - Exists on client instance as `client.realtime`\n   - Thin wrapper over [WebSocket](https://developer.mozilla.org/en-US/docs/Web/API/WebSocket)\n   - Use this for connecting to the API, authenticating, and sending items\n   - There is **no item validation**, you will have to rely on the API specification directly\n   - Dispatches events as `server.{event_name}` and `client.{event_name}`, respectively\n1. [`RealtimeConversation`](./lib/conversation.js)\n   - Exists on client instance as `client.conversation`\n   - Stores a client-side cache of your current conversation\n   - Has **event validation**, will validate incoming events to make sure it can cache them properly\n\n# Using the reference client\n\nThe client comes packaged with some basic utilities that make it easy to build realtime\napps quickly.\n\n## Sending messages\n\nSending messages to the server from the user is easy.\n\n```javascript\nclient.sendUserMessageContent([{ type: 'input_text', text: `How are you?` }]);\n// or (empty audio)\nclient.sendUserMessageContent([\n  { type: 'input_audio', audio: new Int16Array(0) },\n]);\n```\n\n## Sending streaming audio\n\nTo send streaming audio, use the `.appendInputAudio()` method. If you're in `turn_detection: 'disabled'` mode,\nthen you need to use `.createResponse()` to tell the model to respond.\n\n```javascript\n// Send user audio, must be Int16Array or ArrayBuffer\n// Default audio format is pcm16 with sample rate of 24,000 Hz\n// This populates 1s of noise in 0.1s chunks\nfor (let i = 0; i < 10; i++) {\n  const data = new Int16Array(2400);\n  for (let n = 0; n < 2400; n++) {\n    const value = Math.floor((Math.random() * 2 - 1) * 0x8000);\n    data[n] = value;\n  }\n  client.appendInputAudio(data);\n}\n// Pending audio is committed and model is asked to generate\nclient.createResponse();\n```\n\n## Adding and using tools\n\nWorking with tools is easy. Just call `.addTool()` and set a callback as the second parameter.\nThe callback will be executed with the parameters for the tool, and the result will be automatically\nsent back to the model.\n\n```javascript\n// We can add tools as well, with callbacks specified\nclient.addTool(\n  {\n    name: 'get_weather',\n    description:\n      'Retrieves the weather for a given lat, lng coordinate pair. Specify a label for the location.',\n    parameters: {\n      type: 'object',\n      properties: {\n        lat: {\n          type: 'number',\n          description: 'Latitude',\n        },\n        lng: {\n          type: 'number',\n          description: 'Longitude',\n        },\n        location: {\n          type: 'string',\n          description: 'Name of the location',\n        },\n      },\n      required: ['lat', 'lng', 'location'],\n    },\n  },\n  async ({ lat, lng, location }) => {\n    const result = await fetch(\n      `https://api.open-meteo.com/v1/forecast?latitude=${lat}&longitude=${lng}&current=temperature_2m,wind_speed_10m`,\n    );\n    const json = await result.json();\n    return json;\n  },\n);\n```\n\n### Manually using tools\n\nThe `.addTool()` method automatically runs a tool handler and triggers a response\non handler completion. Sometimes you may not want that, for example: using tools\nto generate a schema that you use for other purposes.\n\nIn this case, we can use the `tools` item with `updateSession`. In this case you\n**must** specify `type: 'function'`, which is not required for `.addTool()`.\n\n**Note:** Tools added with `.addTool()` will **not** be overridden when updating\nsessions manually like this, but every `updateSession()` change will override previous\n`updateSession()` changes. Tools added via `.addTool()` are persisted and appended\nto anything set manually here.\n\n```javascript\nclient.updateSession({\n  tools: [\n    {\n      type: 'function',\n      name: 'get_weather',\n      description:\n        'Retrieves the weather for a given lat, lng coordinate pair. Specify a label for the location.',\n      parameters: {\n        type: 'object',\n        properties: {\n          lat: {\n            type: 'number',\n            description: 'Latitude',\n          },\n          lng: {\n            type: 'number',\n            description: 'Longitude',\n          },\n          location: {\n            type: 'string',\n            description: 'Name of the location',\n          },\n        },\n        required: ['lat', 'lng', 'location'],\n      },\n    },\n  ],\n});\n```\n\nThen, to handle function calls...\n\n```javascript\nclient.on('conversation.updated', ({ item, delta }) => {\n  if (item.type === 'function_call') {\n    // do something\n    if (delta.arguments) {\n      // populating the arguments\n    }\n  }\n});\n\nclient.on('conversation.item.completed', ({ item }) => {\n  if (item.type === 'function_call') {\n    // your function call is complete, execute some custom code\n  }\n});\n```\n\n## Interrupting the model\n\nYou may want to manually interrupt the model, especially in `turn_detection: 'disabled'` mode.\nTo do this, we can use:\n\n```javascript\n// id is the id of the item currently being generated\n// sampleCount is the number of audio samples that have been heard by the listener\nclient.cancelResponse(id, sampleCount);\n```\n\nThis method will cause the model to immediately cease generation, but also truncate the\nitem being played by removing all audio after `sampleCount` and clearing the text\nresponse. By using this method you can interrupt the model and prevent it from \"remembering\"\nanything it has generated that is ahead of where the user's state is.\n\n# Client events\n\nIf you need more manual control and want to send custom client events according\nto the [Realtime Client Events API Reference](https://platform.openai.com/docs/api-reference/realtime-client-events),\nyou can use `client.realtime.send()` like so:\n\n```javascript\n// manually send a function call output\nclient.realtime.send('conversation.item.create', {\n  item: {\n    type: 'function_call_output',\n    call_id: 'my-call-id',\n    output: '{function_succeeded:true}',\n  },\n});\nclient.realtime.send('response.create');\n```\n\n## Reference client utility events\n\nWith `RealtimeClient` we have reduced the event overhead from server events to **five**\nmain events that are most critical for your application control flow. These events\n**are not** part of the API specification itself, but wrap logic to make application\ndevelopment easier.\n\n```javascript\n// errors like connection failures\nclient.on('error', (event) => {\n  // do thing\n});\n\n// in VAD mode, the user starts speaking\n// we can use this to stop audio playback of a previous response if necessary\nclient.on('conversation.interrupted', () => {\n  /* do something */\n});\n\n// includes all changes to conversations\n// delta may be populated\nclient.on('conversation.updated', ({ item, delta }) => {\n  // get all items, e.g. if you need to update a chat window\n  const items = client.conversation.getItems();\n  switch (item.type) {\n    case 'message':\n      // system, user, or assistant message (item.role)\n      break;\n    case 'function_call':\n      // always a function call from the model\n      break;\n    case 'function_call_output':\n      // always a response from the user / application\n      break;\n  }\n  if (delta) {\n    // Only one of the following will be populated for any given event\n    // delta.audio = Int16Array, audio added\n    // delta.transcript = string, transcript added\n    // delta.arguments = string, function arguments added\n  }\n});\n\n// only triggered after item added to conversation\nclient.on('conversation.item.appended', ({ item }) => {\n  /* item status can be 'in_progress' or 'completed' */\n});\n\n// only triggered after item completed in conversation\n// will always be triggered after conversation.item.appended\nclient.on('conversation.item.completed', ({ item }) => {\n  /* item status will always be 'completed' */\n});\n```\n\n# Server events\n\nIf you want more control over your application development, you can use the\n`realtime.event` event and choose only to respond to **server** events.\nThe full documentation for these events are available on\nthe [Realtime Server Events API Reference](https://platform.openai.com/docs/api-reference/realtime-server-events).\n\n```javascript\n// all events, can use for logging, debugging, or manual event handling\nclient.on('realtime.event', ({ time, source, event }) => {\n  // time is an ISO timestamp\n  // source is 'client' or 'server'\n  // event is the raw event payload (json)\n  if (source === 'server') {\n    doSomething(event);\n  }\n});\n```\n\n# Running tests\n\nYou will need to make sure you have a `.env` file with `OPENAI_API_KEY=` set in order\nto run tests. From there, running the test suite is easy.\n\n```shell\n$ npm test\n```\n\nTo run tests with debug logs (will log events sent to and received from WebSocket), use:\n\n```shell\n$ npm test -- --debug\n```\n\n# Acknowledgements and contact\n\nThank you for checking out the Realtime API. Would love to hear from you.\nSpecial thanks to the Realtime API team for making this all possible.\n\n- OpenAI Developers / [@OpenAIDevs](https://x.com/OpenAIDevs)\n- Jordan Sitkin / API / [@dustmason](https://x.com/dustmason)\n- Mark Hudnall / API / [@landakram](https://x.com/landakram)\n- Peter Bakkum / API / [@pbbakkum](https://x.com/pbbakkum)\n- Atty Eleti / API / [@athyuttamre](https://x.com/athyuttamre)\n- Jason Clark / API / [@onebitToo](https://x.com/onebitToo)\n- Keith Horwood / API + DX / [@keithwhor](https://x.com/keithwhor)\n","readmeFilename":"README.md"}