Integrate Omni with FreeSWITCH

<include>
  <extension name="omni-agent">
    <condition field="destination_number" expression="^9000$">
      <action application="answer"/>
      <action application="audio_stream"
              data="start wss://${bridge_host}/fork?uuid=${uuid} mono 16k"/>
      <action application="park"/>
    </condition>
  </extension>
</include>

import "dotenv/config";
import Fastify from "fastify";
import websocket from "@fastify/websocket";
import WebSocket from "ws";
import esl from "modesl";

const { PYAI_API_KEY, FS_HOST, FS_PASSWORD } = process.env;

const app = Fastify();
await app.register(websocket);

const calls = new Map(); // uuid → { omni, fsWS }

// Zero-state: no agent_id needed. Add &session_label=<tag> to brand the call.
const omniURL = "wss://api.pyai.com/v1/omni?format=pcm16&rate=16000";

app.get("/fork", { websocket: true }, (fsWS, req) => {
  const uuid = new URL(`http://x${req.url}`).searchParams.get("uuid");
  const omni = new WebSocket(omniURL, [`pyai-key.${PYAI_API_KEY}`]);
  calls.set(uuid, { omni, fsWS });

  // caller audio (L16 16 kHz) → Omni (PCM16 16 kHz): identical bytes
  fsWS.on("message", (data, isBinary) => {
    if (isBinary && omni.readyState === WebSocket.OPEN) omni.send(data);
  });

  // agent audio → back into the channel; text frames are session events
  omni.on("message", (data, isBinary) => {
    if (isBinary) fsWS.send(data);
    else handleOmniEvent(JSON.parse(data.toString()), uuid);
  });

  const cleanup = () => { calls.delete(uuid); omni.close(); };
  fsWS.on("close", cleanup);
  omni.on("close", () => fsWS.close());
});

const control = new esl.Connection(FS_HOST, 8021, FS_PASSWORD, () => {
  control.subscribe(["DTMF"]);
});

control.on("esl::event::DTMF::*", (e) => {
  const uuid = e.getHeader("Unique-ID");
  const digit = e.getHeader("DTMF-Digit");
  const call = calls.get(uuid);
  if (call) forwardDtmf(call.omni, digit);
});

function forwardDtmf(omni, digit) {
  // Localized: exact dtmf frame per the Omni wire protocol reference.
  if (omni.readyState === WebSocket.OPEN) {
    omni.send(JSON.stringify({ type: "dtmf", digit }));
  }
}

function handleOmniEvent(evt, uuid) {
  // Omni server -> client frames are keyed on `event` (NOT `type`, your
  // outbound configure/dtmf are `type`-keyed). Switching on `evt.type` is
  // the #1 Omni bug: every case misses and the agent appears mute.
  switch (evt.event) {
    case "hello":
    case "session_started":
    case "configured":
      break;

    case "flush":
    case "barge_in":
      // Barge-in: stop the agent audio currently playing into the channel.
      control.api("uuid_break", uuid);
      break;

    case "transfer_to_human":
      // Hand the live channel off to a human. `destination` comes from the
      // tool's per-agent setting; fall back to your default extension.
      control.api("uuid_transfer", `${uuid} ${evt.destination || process.env.HUMAN_EXTENSION} XML default`);
      break;

    case "send_dtmf":
      control.api("uuid_send_dtmf", `${uuid} ${evt.digits}`);
      break;

    case "play_hold":
      // Play hold audio; it stops when the next agent audio arrives.
      control.api("uuid_broadcast", `${uuid} local_stream://moh aleg`);
      break;

    case "end_call":
      control.api("uuid_kill", uuid);
      break;

    case "collect":
      // Fire-and-forget: the value returns via normal speech (STT) or the
      // DTMF you already forward inbound. Nothing to do for most agents.
      break;

    case "session_end":
      calls.get(uuid)?.fsWS.close();
      break;
  }
}

app.listen({ port: 8080, host: "0.0.0.0" });