Stream speech-to-text in real time

const FRAME = 320; // 20 ms @ 16 kHz

class CaptureProcessor extends AudioWorkletProcessor {
  constructor() {
    super();
    this._buf = new Int16Array(FRAME);
    this._n = 0;
  }
  process(inputs) {
    const ch = inputs[0]?.[0];
    if (!ch) return true;
    for (let i = 0; i < ch.length; i++) {
      const s = Math.max(-1, Math.min(1, ch[i]));
      this._buf[this._n++] = s < 0 ? s * 0x8000 : s * 0x7fff;
      if (this._n === FRAME) {
        const out = this._buf.slice();
        this.port.postMessage(out.buffer, [out.buffer]);
        this._n = 0;
      }
    }
    return true;
  }
}
registerProcessor("capture-processor", CaptureProcessor);

const finalsByUtterance = new Map(); // committed lines keyed by utterance_id
let pending = "";      // current partial (greyed)

function onPartial(text) {
  pending = text;      // replace, don't append
  paint();
}

function onFinal(utteranceId, text) {
  finalsByUtterance.set(utteranceId, text); // replace the earlier speech_final
  pending = "";        // clear the greyed line
  paint();
}

function paint() {
  els.committed.textContent = [...finalsByUtterance.values()].join(" ");
  els.pending.textContent = pending;      // styled grey/italic in CSS
}

const HEAR_STREAM_URL =
  "wss://api.pyai.com/v1/audio/transcriptions/stream" +
  "?protocol=pyai-hear-v1&model=pyai-hear&language=auto&sample_rate=16000&encoding=pcm16&interim_results=true&endpointing_ms=800";

function connectHearStream(apiKey, capture) {
  const ws = new WebSocket(HEAR_STREAM_URL, ["pyai.v1", `pyai-key.${apiKey}`]);
  ws.binaryType = "arraybuffer";

  // mic/call PCM16 frames → server (binary)
  capture.port.onmessage = (e) => {
    if (ws.readyState === WebSocket.OPEN) ws.send(e.data);
  };

  // text frames are JSON results, all framing lives in handleFrame()
  ws.onmessage = (e) => {
    if (typeof e.data === "string") handleFrame(JSON.parse(e.data));
  };
  return ws;
}

// Force-finalize the current utterance (e.g. on your own end-of-turn signal).
// The explicit finalization frame is JSON: {"type":"commit"}.
function commit(ws) {
  if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: "commit" }));
}

// Change the minimum trailing-pause floor without reconnecting.
function setEndpointing(ws, endpointingMs) {
  if (ws.readyState === WebSocket.OPEN) {
    ws.send(JSON.stringify({ type: "config", endpointing_ms: endpointingMs }));
  }
}

// The one place that knows the frame schema. The engine emits bare types:
// partial / partial_stable (interim) and speech_final / final (stable).
function handleFrame(msg) {
  switch (msg.type) {
    case "config_ack":
      if (msg.warnings.length > 0) {
        console.error("endpointing config warning:", msg.warnings);
      }
      break;
    case "partial":
    case "partial_stable":
      onPartial(msg.text);
      break;
    case "speech_final":
      console.info("endpoint reason:", msg.endpoint_reason);
      // Stable utterance result, followed by the corrected full-context `final`.
      // Show it immediately, but do not append it as a second committed row.
      onPartial(msg.text);
      break;
    case "final":
      onFinal(msg.utterance_id, msg.text);
      break;
    case "error":
      console.error("hear-stream error:", msg.code, msg.message);
      break;
  }
}