Build a browser voice agent

const API_KEY = "pyai_test_..."; // publishable/short-lived token in prod

// Omni is zero-state, no agent to create. Optionally add
// &session_label=<tag> to brand the session in your kb_endpoint.
const url = "wss://api.pyai.com/v1/omni?format=pcm16&rate=24000";

async function start() {
  const ctx = new AudioContext({ sampleRate: 24000 });
  await ctx.audioWorklet.addModule("capture-processor.js");
  await ctx.audioWorklet.addModule("playback-processor.js");

  const stream = await navigator.mediaDevices.getUserMedia({
    audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
  });

  const src = ctx.createMediaStreamSource(stream);
  const capture = new AudioWorkletNode(ctx, "capture-processor");
  const playback = new AudioWorkletNode(ctx, "playback-processor");
  src.connect(capture);          // mic → capture (no audible monitor)
  playback.connect(ctx.destination); // playback → speakers

  const ws = new WebSocket(url, [`pyai-key.${API_KEY}`]);
  ws.binaryType = "arraybuffer";

  // set the agent's voice + behavior on connect (zero-state configure)
  ws.onopen = () => ws.send(JSON.stringify({
    type: "configure",
    voice_id: "stock_dorit_en_us",
    persona: "You are a friendly receptionist.",
  }));

  // mic frames → server
  capture.port.onmessage = (e) => {
    if (ws.readyState === WebSocket.OPEN) ws.send(e.data);
  };

  ws.onmessage = (e) => {
    if (typeof e.data === "string") {
      handleEvent(JSON.parse(e.data), playback); // see next step
    } else {
      playback.port.postMessage(e.data); // agent audio → speakers
    }
  };

  ws.onclose = (e) => console.log("closed", e.code, e.reason);
  window.stop = () => { ws.close(); stream.getTracks().forEach((t) => t.stop()); };
}

import PyAI from "@pyai/sdk";

const pyai = new PyAI({ apiKey: API_KEY });
// omni.connect() handles the frame-key asymmetry for you: it sends a
// type-keyed `configure` on open and parses event-keyed server frames into
// callbacks, so you can't trip the "configured but zero turns" bug.
const omni = pyai.omni.connect({
  rate: 24000, // zero-state, sessionLabel is optional, not required
  configure: { voice_id: "stock_dorit_en_us", persona: "You are a friendly receptionist." },
  onAudio: (chunk) => playback.port.postMessage(chunk), // agent audio → speakers
  onBargeIn: () => playback.port.postMessage("clear"),  // drop stale audio
  onTranscript: (f) => renderTranscript(f),
  onError: (e) => console.error(e),
});
// mic frames → server
capture.port.onmessage = (e) => omni.sendAudio(e.data);