OpenAI Realtime API Guide 2025: Low-Latency Voice & Audio

Raw WebSocket connection

import websocket
import json
import os

url = "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview"

headers = [
    f"Authorization: Bearer {os.environ['OPENAI_API_KEY']}",
    "OpenAI-Beta: realtime=v1",
]

ws = websocket.WebSocketApp(
    url,
    header=headers,
    on_open=on_open,
    on_message=on_message,
    on_error=on_error,
    on_close=on_close,
)
ws.run_forever()

pip install openai

pip install "openai>=1.50.0"

realtime_quickstart.py

import asyncio
from openai import AsyncOpenAI

client = AsyncOpenAI()

async def main():
    async with client.beta.realtime.connect(
        model="gpt-4o-realtime-preview"
    ) as conn:
        # Configure the session
        await conn.session.update(session={
            "voice": "alloy",
            "instructions": "You are a helpful assistant. Respond concisely.",
            "turn_detection": {"type": "server_vad"},
        })

        # Send a text message (audio works the same way)
        await conn.conversation.item.create(item={
            "type": "message",
            "role": "user",
            "content": [{"type": "input_text", "text": "Say hello!"}],
        })
        await conn.response.create()

        # Stream the response events
        async for event in conn:
            if event.type == "response.audio.delta":
                # event.delta is base64-encoded PCM16 audio
                audio_bytes = event.delta
                # write to speaker or buffer here
                print(f"Audio chunk received: {len(audio_bytes)} bytes")
            elif event.type == "response.done":
                print("Response complete")
                break

asyncio.run(main())

Browser WebSocket + MediaRecorder pattern

// 1. Fetch ephemeral token from your backend
const tokenRes = await fetch("/api/realtime-token");
const { client_secret } = await tokenRes.json();

// 2. Connect via WebSocket using the ephemeral token
const ws = new WebSocket(
  "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview",
  ["realtime", `openai-insecure-api-key.${client_secret.value}`]
);

ws.onopen = () => {
  // Configure session
  ws.send(JSON.stringify({
    type: "session.update",
    session: {
      voice: "shimmer",
      instructions: "You are a helpful assistant.",
      turn_detection: { type: "server_vad" },
    }
  }));
};

// 3. Capture microphone audio with MediaRecorder
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const recorder = new MediaRecorder(stream, { mimeType: "audio/webm" });

recorder.ondataavailable = async (e) => {
  const buffer = await e.data.arrayBuffer();
  const base64 = btoa(String.fromCharCode(...new Uint8Array(buffer)));
  ws.send(JSON.stringify({
    type: "input_audio_buffer.append",
    audio: base64,
  }));
};

recorder.start(100); // send chunks every 100ms

// 4. Handle incoming audio deltas
ws.onmessage = (e) => {
  const event = JSON.parse(e.data);
  if (event.type === "response.audio.delta") {
    // Decode base64 PCM16 and play via AudioContext
    playAudioChunk(event.delta);
  }
};

session.update event — full config

{
  "type": "session.update",
  "session": {
    // Voice options: alloy, echo, shimmer, verse, ash, ballad, coral, sage
    "voice": "alloy",

    // System prompt for the voice model
    "instructions": "You are a friendly customer support agent for Acme Corp.",

    // Input audio format: pcm16, g711_ulaw, g711_alaw
    "input_audio_format": "pcm16",

    // Output audio format: pcm16, g711_ulaw, g711_alaw
    "output_audio_format": "pcm16",

    // server_vad: automatic turn detection
    // null: manual push-to-talk (use input_audio_buffer.commit)
    "turn_detection": {
      "type": "server_vad",
      "threshold": 0.5,        // VAD sensitivity (0.0-1.0)
      "prefix_padding_ms": 300, // audio before speech start to include
      "silence_duration_ms": 500 // silence before triggering response
    },

    // Modalities: ["text", "audio"] or ["text"] for text-only
    "modalities": ["text", "audio"],

    // Temperature for response generation
    "temperature": 0.8,

    // Max output tokens (audio + text combined)
    "max_response_output_tokens": 4096
  }
}

Add tools to session

{
  "type": "session.update",
  "session": {
    "tools": [
      {
        "type": "function",
        "name": "get_weather",
        "description": "Get current weather for a city",
        "parameters": {
          "type": "object",
          "properties": {
            "location": {
              "type": "string",
              "description": "City name, e.g. San Francisco"
            }
          },
          "required": ["location"]
        }
      }
    ],
    "tool_choice": "auto"
  }
}

Handle function call and return result

ws.onmessage = async (e) => {
  const event = JSON.parse(e.data);

  if (event.type === "response.function_call_arguments.done") {
    const args = JSON.parse(event.arguments);
    const callId = event.call_id;

    // Execute your function
    const result = await getWeather(args.location);

    // Return the result to the model
    ws.send(JSON.stringify({
      type: "conversation.item.create",
      item: {
        type: "function_call_output",
        call_id: callId,
        output: JSON.stringify(result),
      }
    }));

    // Ask the model to continue the conversation
    ws.send(JSON.stringify({ type: "response.create" }));
  }
};