Build a phone voice agent with Twilio

import "dotenv/config";
import Fastify from "fastify";
import websocket from "@fastify/websocket";
import WebSocket from "ws";
import { mulaw } from "alawmulaw";
import twilio from "twilio";

const { PUBLIC_HOST, PYAI_API_KEY } = process.env;

const app = Fastify();
await app.register(websocket);

app.post("/voice", (req, reply) => {
  const twiml = `
    <Response>
      <Connect>
        <Stream url="wss://${PUBLIC_HOST}/media" />
      </Connect>
    </Response>`;
  reply.type("text/xml").send(twiml.trim());
});

// Zero-state: no agent_id needed. Add &session_label=<tag> to brand the call.
const omniURL = "wss://api.pyai.com/v1/omni?format=pcm16&rate=8000";

app.get("/media", { websocket: true }, (twilioWS) => {
  let streamSid = null;
  let callSid = null;

  const omni = new WebSocket(omniURL, [`pyai-key.${PYAI_API_KEY}`]);

  twilioWS.on("message", (raw) => {
    const msg = JSON.parse(raw.toString());
    switch (msg.event) {
      case "start":
        streamSid = msg.start.streamSid;
        callSid = msg.start.callSid;
        break;

      case "media": {
        // base64 μ-law (8 kHz) → Int16 PCM (8 kHz) → Omni (0x01 + binary)
        const ulaw = Buffer.from(msg.media.payload, "base64");
        const pcm = mulaw.decode(ulaw); // Int16Array
        if (omni.readyState === WebSocket.OPEN) {
          const body = Buffer.from(pcm.buffer, pcm.byteOffset, pcm.byteLength);
          omni.send(Buffer.concat([Buffer.from([0x01]), body]));
        }
        break;
      }

      case "dtmf":
        // Caller pressed a key. Forward the digit to Omni so the agent can
        // react (exact dtmf frame: see the protocol reference).
        forwardDtmf(omni, msg.dtmf.digit);
        break;

      case "stop":
        omni.close();
        break;
    }
  });

  // ...agent → caller wiring in the next step (same scope)

  omni.on("message", (data, isBinary) => {
    if (isBinary) {
      // Omni PCM16 (8 kHz) → μ-law → base64 → Twilio
      const pcm = new Int16Array(
        data.buffer, data.byteOffset, data.byteLength / 2,
      );
      const ulaw = mulaw.encode(pcm); // Uint8Array
      twilioWS.send(JSON.stringify({
        event: "media",
        streamSid,
        media: { payload: Buffer.from(ulaw).toString("base64") },
      }));
    } else {
      handleOmniEvent(JSON.parse(data.toString()), { twilioWS, streamSid, callSid });
    }
  });

  twilioWS.on("close", () => omni.close());
  omni.on("close", () => twilioWS.close());
});

function forwardDtmf(omni, digit) {
  // Localized: shape per the Omni wire protocol reference.
  if (omni.readyState === WebSocket.OPEN) {
    omni.send(JSON.stringify({ type: "dtmf", digit }));
  }
}

function handleOmniEvent(evt, { twilioWS, streamSid, callSid }) {
  // Omni server -> client frames are keyed on `event` (NOT `type`, your
  // outbound configure/dtmf are `type`-keyed). Don't conflate this with the
  // Twilio media-stream messages above, which are also `event`-keyed.
  switch (evt.event) {
    case "hello":
    case "session_started":
    case "configured":
      break;

    case "flush":
    case "barge_in":
      // Barge-in: flush Twilio's outbound buffer so the agent stops mid-word.
      twilioWS.send(JSON.stringify({ event: "clear", streamSid }));
      break;

    case "transfer_to_human":
      transferCall(callSid, evt.destination); // redirect the live call to a person (destination from the agent's tool setting)
      break;

    case "send_dtmf":
      twilioWS.send(JSON.stringify({ event: "dtmf", streamSid, dtmf: { digits: evt.digits } }));
      break;

    case "play_hold":
      // Play hold audio (e.g. redirect to TwiML <Play>/<Enqueue>); it ends
      // when the next agent audio arrives.
      break;

    case "end_call":
      twilioWS.close(); // and hang up the call leg via the Twilio REST API
      break;

    case "collect":
      // Fire-and-forget: the value returns via normal speech (STT) or the
      // caller DTMF you already forward. Nothing to do for most agents.
      break;

    case "session_end":
      twilioWS.close();
      break;
  }
}

const rest = twilio(process.env.TWILIO_ACCOUNT_SID, process.env.TWILIO_AUTH_TOKEN);

function transferCall(callSid) {
  // Replace the live call's TwiML with a <Dial> to a human.
  rest.calls(callSid).update({
    twiml: `<Response><Dial>${process.env.HUMAN_NUMBER}</Dial></Response>`,
  });
}

app.listen({ port: 8080, host: "0.0.0.0" });