Build a phone voice agent with Twilio

import "dotenv/config";
import Fastify from "fastify";
import websocket from "@fastify/websocket";
import WebSocket from "ws";
import { mulaw } from "alawmulaw";
import twilio from "twilio";

const { PUBLIC_HOST, PYAI_API_KEY } = process.env;

const app = Fastify();
await app.register(websocket);

app.post("/voice", (req, reply) => {
  const twiml = `
    <Response>
      <Connect>
        <Stream url="wss://${PUBLIC_HOST}/media" />
      </Connect>
    </Response>`;
  reply.type("text/xml").send(twiml.trim());
});

// Set PYAI_AGENT_ID to load a stored production Agent profile. Omit it for
// a fully inline session; an unrelated session_label remains an opaque tag.
const label = process.env.PYAI_AGENT_ID
  ? `&session_label=${encodeURIComponent(process.env.PYAI_AGENT_ID)}`
  : "";
const omniURL = `wss://api.pyai.com/v1/omni?format=pcm16&rate=8000${label}`;

function omniControl(body) {
  return Buffer.concat([
    Buffer.from([0x03]),
    Buffer.from(JSON.stringify(body), "utf8"),
  ]);
}

app.get("/media", { websocket: true }, (twilioWS) => {
  let streamSid = null;
  let callSid = null;

  const omni = new WebSocket(omniURL, ["pyai.v1", `pyai-key.${PYAI_API_KEY}`]);
  omni.on("open", () => {
    const configure = { type: "configure" };
    if (!process.env.PYAI_AGENT_ID) {
      configure.voice_id = "stock_dorit_en_us";
      configure.persona = "You are a concise phone receptionist.";
    }
    omni.send(omniControl(configure));
  });

  twilioWS.on("message", (raw) => {
    const msg = JSON.parse(raw.toString());
    switch (msg.event) {
      case "start":
        streamSid = msg.start.streamSid;
        callSid = msg.start.callSid;
        break;

      case "media": {
        // base64 μ-law (8 kHz) → Int16 PCM (8 kHz) → Omni (0x01 + binary)
        const ulaw = Buffer.from(msg.media.payload, "base64");
        const pcm = mulaw.decode(ulaw); // Int16Array
        if (omni.readyState === WebSocket.OPEN) {
          const body = Buffer.from(pcm.buffer, pcm.byteOffset, pcm.byteLength);
          omni.send(Buffer.concat([Buffer.from([0x01]), body]));
        }
        break;
      }

      case "dtmf":
        // Caller pressed a key. Forward the digit to Omni so the agent can
        // react (exact dtmf frame: see the protocol reference).
        forwardDtmf(omni, msg.dtmf.digit);
        break;

      case "stop":
        omni.close();
        break;
    }
  });

  // ...agent → caller wiring in the next step (same scope)

  omni.on("message", (data, isBinary) => {
    if (!isBinary) return;
    const frame = Buffer.from(data);
    const tag = frame[0];
    const body = frame.subarray(1);
    if (tag === 0x01) {
      // Omni PCM16 (8 kHz) → μ-law → base64 → Twilio
      const pcmBytes = Uint8Array.from(body); // aligned copy after tag
      const pcm = new Int16Array(pcmBytes.buffer);
      const ulaw = mulaw.encode(pcm); // Uint8Array
      twilioWS.send(JSON.stringify({
        event: "media",
        streamSid,
        media: { payload: Buffer.from(ulaw).toString("base64") },
      }));
    } else if (tag === 0x02) {
      // Caller transcript deltas are plain UTF-8, never JSON.
      console.log("caller transcript delta:", body.toString("utf8"));
    } else if (tag === 0x03) {
      handleOmniEvent(JSON.parse(body.toString("utf8")), { twilioWS, streamSid, callSid });
    }
  });

  twilioWS.on("close", () => omni.close());
  omni.on("close", () => twilioWS.close());
});

function forwardDtmf(omni, digit) {
  // Localized: shape per the Omni wire protocol reference.
  if (omni.readyState === WebSocket.OPEN) {
    omni.send(omniControl({ type: "dtmf", digit }));
  }
}

function handleOmniEvent(evt, { twilioWS, streamSid, callSid }) {
  // Omni server -> client frames are keyed on `event` (NOT `type`, your
  // outbound configure/dtmf are `type`-keyed). Don't conflate this with the
  // Twilio media-stream messages above, which are also `event`-keyed.
  switch (evt.event) {
    case "hello":
    case "session_started":
    case "configured":
      break;

    case "flush":
      // Barge-in: flush Twilio's outbound buffer so the agent stops mid-word.
      twilioWS.send(JSON.stringify({ event: "clear", streamSid }));
      break;

    case "transfer_to_human":
      transferCall(callSid, evt.destination); // redirect the live call to a person (destination from the agent's tool setting)
      break;

    case "send_dtmf":
      twilioWS.send(JSON.stringify({ event: "dtmf", streamSid, dtmf: { digits: evt.digits } }));
      break;

    case "play_hold":
      // Play hold audio (e.g. redirect to TwiML <Play>/<Enqueue>); it ends
      // when the next agent audio arrives.
      break;

    case "end_call":
      twilioWS.close(); // and hang up the call leg via the Twilio REST API
      break;

    case "collect":
      // Fire-and-forget: the value returns via normal speech (STT) or the
      // caller DTMF you already forward. Nothing to do for most agents.
      break;

    case "session_end":
      twilioWS.close();
      break;
  }
}

const rest = twilio(process.env.TWILIO_ACCOUNT_SID, process.env.TWILIO_AUTH_TOKEN);

function transferCall(callSid, destination) {
  // Replace the live call's TwiML with a <Dial> to a human.
  const target = destination || process.env.HUMAN_NUMBER;
  rest.calls(callSid).update({
    twiml: `<Response><Dial>${target}</Dial></Response>`,
  });
}

app.listen({ port: 8080, host: "0.0.0.0" });