sailfork / t3code-github / packages/ client-runtime/ src/ voice-input/ stream.ts

// @effect-diagnostics globalTimers:off - imperative WebSocket callbacks own and clear their response deadline.
import {
  SPEECH_STREAM_MAX_CHUNK_BYTES,
  SPEECH_STREAM_MAX_QUEUED_BYTES,
  SpeechStreamEvent,
  type SpeechStreamText,
  type SpeechTranscriptionOptions,
} from "@t3tools/contracts";
import * as Schema from "effect/Schema";

const decodeEvent = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechStreamEvent));

/** Audio stays bounded while the environment acknowledges a small send window. */
export function openSpeechStream(input: {
  readonly url: string;
  readonly options: SpeechTranscriptionOptions;
  readonly signal: AbortSignal;
  readonly canSend: () => boolean;
  readonly onText: (text: SpeechStreamText) => void;
  readonly onError: (error: Error) => void;
  readonly createSocket?: (url: string) => WebSocket;
}) {
  input.signal.throwIfAborted();
  const socket = (input.createSocket ?? ((url) => new WebSocket(url)))(input.url);
  let resolveFinal!: (text: string) => void;
  let rejectFinal!: (error: Error) => void;
  const final = new Promise<string>((resolve, reject) => {
    resolveFinal = resolve;
    rejectFinal = reject;
  });
  // A capture failure can precede stop(), when nobody is awaiting the final result yet.
  void final.catch(() => undefined);
  let queue: Uint8Array<ArrayBuffer>[] = [];
  let queuedBytes = 0;
  const inFlight: number[] = [];
  let inFlightBytes = 0;
  let finishing = false;
  let finishSent = false;
  let closed = false;
  let isReady = false;
  let revision = -1;
  let timer: ReturnType<typeof setTimeout>;
  const cleanup = () => {
    clearTimeout(timer);
    input.signal.removeEventListener("abort", abort);
    queue = [];
    queuedBytes = 0;
  };
  const fail = (error: Error, notify = true) => {
    if (closed) return;
    closed = true;
    cleanup();
    rejectFinal(error);
    socket.close();
    if (notify) input.onError(error);
  };
  const armTimeout = () => {
    clearTimeout(timer);
    timer = setTimeout(() => fail(new Error("Live transcription stopped responding.")), 120_000);
  };
  const abort = () => fail(new Error("Voice transcription was cancelled."), false);
  const pump = () => {
    if (!isReady || closed || finishSent) return;
    if (!input.canSend()) return fail(new Error("Voice consent was revoked."));
    while (inFlight.length < 2 && queue.length) {
      const chunk = queue.shift()!;
      inFlight.push(chunk.byteLength);
      inFlightBytes += chunk.byteLength;
      queuedBytes -= chunk.byteLength;
      socket.send(chunk);
      armTimeout();
    }
    if (finishing && !queue.length && !inFlight.length) {
      finishSent = true;
      socket.send(JSON.stringify({ type: "finish" }));
      armTimeout();
    }
  };
  socket.addEventListener("open", () => {
    if (!input.canSend()) return fail(new Error("Voice consent was revoked."));
    if (!closed) socket.send(JSON.stringify({ type: "start", options: input.options }));
  });
  socket.addEventListener("message", (event) => {
    if (closed) return;
    if (!input.canSend()) return fail(new Error("Voice consent was revoked."));
    try {
      const message = decodeEvent(event.data);
      switch (message.type) {
        case "ready":
          if (isReady) throw new Error("Unexpected speech stream response.");
          isReady = true;
          clearTimeout(timer);
          pump();
          break;
        case "update":
          if (!inFlight.length || message.revision < revision)
            throw new Error("Unexpected speech stream response.");
          revision = message.revision;
          inFlightBytes -= inFlight.shift()!;
          clearTimeout(timer);
          if (message.text) input.onText(message.text);
          pump();
          if (inFlight.length) armTimeout();
          break;
        case "finished":
          if (!finishSent) throw new Error("Unexpected speech stream response.");
          closed = true;
          cleanup();
          resolveFinal(message.text);
          socket.close();
          break;
        case "error":
          fail(new Error(message.message));
      }
    } catch (error) {
      fail(error instanceof Error ? error : new Error("Invalid speech stream response."));
    }
  });
  socket.addEventListener("close", () =>
    fail(new Error("The voice connection closed. Please record again.")),
  );
  socket.addEventListener("error", () =>
    fail(new Error("Could not connect to live transcription.")),
  );
  input.signal.addEventListener("abort", abort, { once: true });
  armTimeout();
  if (input.signal.aborted) abort();
  return {
    feed: (pcm: Float32Array) => {
      if (closed || finishing) return;
      const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
      if (queuedBytes + inFlightBytes + bytes.byteLength > SPEECH_STREAM_MAX_QUEUED_BYTES) {
        fail(new Error("Voice recordings are limited to five minutes."));
        return;
      }
      let offset = 0;
      while (offset < bytes.length) {
        const tail = queue.at(-1);
        const available = tail ? SPEECH_STREAM_MAX_CHUNK_BYTES - tail.byteLength : 0;
        if (tail && available > 0) {
          const appended = Math.min(available, bytes.length - offset);
          const merged = new Uint8Array(tail.byteLength + appended);
          merged.set(tail);
          merged.set(bytes.subarray(offset, offset + appended), tail.byteLength);
          queue[queue.length - 1] = merged;
          queuedBytes += appended;
          offset += appended;
          continue;
        }
        const chunk = bytes.slice(offset, offset + SPEECH_STREAM_MAX_CHUNK_BYTES);
        queue.push(chunk);
        queuedBytes += chunk.byteLength;
        offset += chunk.byteLength;
      }
      try {
        pump();
      } catch {
        fail(new Error("Could not send microphone audio."));
      }
    },
    finish: () => {
      finishing = true;
      try {
        pump();
      } catch {
        fail(new Error("Could not finish live transcription."));
      }
      return final;
    },
    cancel: abort,
  };
}