Stream speech-to-text in real time

const FRAME = 320; // 20 ms @ 16 kHz

class CaptureProcessor extends AudioWorkletProcessor {
  constructor() {
    super();
    this._buf = new Int16Array(FRAME);
    this._n = 0;
  }
  process(inputs) {
    const ch = inputs[0]?.[0];
    if (!ch) return true;
    for (let i = 0; i < ch.length; i++) {
      const s = Math.max(-1, Math.min(1, ch[i]));
      this._buf[this._n++] = s < 0 ? s * 0x8000 : s * 0x7fff;
      if (this._n === FRAME) {
        const out = this._buf.slice();
        this.port.postMessage(out.buffer, [out.buffer]);
        this._n = 0;
      }
    }
    return true;
  }
}
registerProcessor("capture-processor", CaptureProcessor);

const finals = [];     // committed lines
let pending = "";      // current partial (greyed)

function onPartial(text) {
  pending = text;      // replace, don't append
  paint();
}

function onFinal(text) {
  finals.push(text);   // commit
  pending = "";        // clear the greyed line
  paint();
}

function paint() {
  els.committed.textContent = finals.join(" ");
  els.pending.textContent = pending;      // styled grey/italic in CSS
}

const HEAR_STREAM_URL =
  "wss://api.pyai.com/v1/audio/transcriptions/stream" +
  "?model=pyai-hear&language=en&sample_rate=16000&encoding=pcm16&interim_results=true";

function connectHearStream(apiKey, capture) {
  const ws = new WebSocket(HEAR_STREAM_URL, [`pyai-key.${apiKey}`]);
  ws.binaryType = "arraybuffer";

  // mic/call PCM16 frames → server (binary)
  capture.port.onmessage = (e) => {
    if (ws.readyState === WebSocket.OPEN) ws.send(e.data);
  };

  // text frames are JSON results, all framing lives in handleFrame()
  ws.onmessage = (e) => {
    if (typeof e.data === "string") handleFrame(JSON.parse(e.data));
  };
  return ws;
}

// Force-finalize the current utterance (e.g. on your own end-of-turn signal).
// The flush is {"type":"commit"}, {"type":"end"} is ignored by the engine.
function commit(ws) {
  if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: "commit" }));
}

// The one place that knows the frame schema. The engine emits bare types:
// partial / partial_stable (interim) and speech_final / final (stable).
function handleFrame(msg) {
  switch (msg.type) {
    case "partial":
    case "partial_stable":
      onPartial(msg.text);
      break;
    case "speech_final":
    case "final":
      onFinal(msg.text);
      break;
    case "error":
      console.error("hear-stream error:", msg.code, msg.message);
      break;
  }
}