// @effect-diagnostics globalTimers:off - imperative WebSocket callbacks own and clear their response deadline.
import {
SPEECH_STREAM_MAX_CHUNK_BYTES,
SPEECH_STREAM_MAX_QUEUED_BYTES,
SpeechStreamEvent,
type SpeechStreamText,
type SpeechTranscriptionOptions,
} from "@t3tools/contracts";
import * as Schema from "effect/Schema";
const decodeEvent = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechStreamEvent));
/** Audio stays bounded while the environment acknowledges a small send window. */
export function openSpeechStream(input: {
readonly url: string;
readonly options: SpeechTranscriptionOptions;
readonly signal: AbortSignal;
readonly canSend: () => boolean;
readonly onText: (text: SpeechStreamText) => void;
readonly onError: (error: Error) => void;
readonly createSocket?: (url: string) => WebSocket;
}) {
input.signal.throwIfAborted();
const socket = (input.createSocket ?? ((url) => new WebSocket(url)))(input.url);
let resolveFinal!: (text: string) => void;
let rejectFinal!: (error: Error) => void;
const final = new Promise<string>((resolve, reject) => {
resolveFinal = resolve;
rejectFinal = reject;
});
// A capture failure can precede stop(), when nobody is awaiting the final result yet.
void final.catch(() => undefined);
let queue: Uint8Array<ArrayBuffer>[] = [];
let queuedBytes = 0;
const inFlight: number[] = [];
let inFlightBytes = 0;
let finishing = false;
let finishSent = false;
let closed = false;
let isReady = false;
let revision = -1;
let timer: ReturnType<typeof setTimeout>;
const cleanup = () => {
clearTimeout(timer);
input.signal.removeEventListener("abort", abort);
queue = [];
queuedBytes = 0;
};
const fail = (error: Error, notify = true) => {
if (closed) return;
closed = true;
cleanup();
rejectFinal(error);
socket.close();
if (notify) input.onError(error);
};
const armTimeout = () => {
clearTimeout(timer);
timer = setTimeout(() => fail(new Error("Live transcription stopped responding.")), 120_000);
};
const abort = () => fail(new Error("Voice transcription was cancelled."), false);
const pump = () => {
if (!isReady || closed || finishSent) return;
if (!input.canSend()) return fail(new Error("Voice consent was revoked."));
while (inFlight.length < 2 && queue.length) {
const chunk = queue.shift()!;
inFlight.push(chunk.byteLength);
inFlightBytes += chunk.byteLength;
queuedBytes -= chunk.byteLength;
socket.send(chunk);
armTimeout();
}
if (finishing && !queue.length && !inFlight.length) {
finishSent = true;
socket.send(JSON.stringify({ type: "finish" }));
armTimeout();
}
};
socket.addEventListener("open", () => {
if (!input.canSend()) return fail(new Error("Voice consent was revoked."));
if (!closed) socket.send(JSON.stringify({ type: "start", options: input.options }));
});
socket.addEventListener("message", (event) => {
if (closed) return;
if (!input.canSend()) return fail(new Error("Voice consent was revoked."));
try {
const message = decodeEvent(event.data);
switch (message.type) {
case "ready":
if (isReady) throw new Error("Unexpected speech stream response.");
isReady = true;
clearTimeout(timer);
pump();
break;
case "update":
if (!inFlight.length || message.revision < revision)
throw new Error("Unexpected speech stream response.");
revision = message.revision;
inFlightBytes -= inFlight.shift()!;
clearTimeout(timer);
if (message.text) input.onText(message.text);
pump();
if (inFlight.length) armTimeout();
break;
case "finished":
if (!finishSent) throw new Error("Unexpected speech stream response.");
closed = true;
cleanup();
resolveFinal(message.text);
socket.close();
break;
case "error":
fail(new Error(message.message));
}
} catch (error) {
fail(error instanceof Error ? error : new Error("Invalid speech stream response."));
}
});
socket.addEventListener("close", () =>
fail(new Error("The voice connection closed. Please record again.")),
);
socket.addEventListener("error", () =>
fail(new Error("Could not connect to live transcription.")),
);
input.signal.addEventListener("abort", abort, { once: true });
armTimeout();
if (input.signal.aborted) abort();
return {
feed: (pcm: Float32Array) => {
if (closed || finishing) return;
const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
if (queuedBytes + inFlightBytes + bytes.byteLength > SPEECH_STREAM_MAX_QUEUED_BYTES) {
fail(new Error("Voice recordings are limited to five minutes."));
return;
}
let offset = 0;
while (offset < bytes.length) {
const tail = queue.at(-1);
const available = tail ? SPEECH_STREAM_MAX_CHUNK_BYTES - tail.byteLength : 0;
if (tail && available > 0) {
const appended = Math.min(available, bytes.length - offset);
const merged = new Uint8Array(tail.byteLength + appended);
merged.set(tail);
merged.set(bytes.subarray(offset, offset + appended), tail.byteLength);
queue[queue.length - 1] = merged;
queuedBytes += appended;
offset += appended;
continue;
}
const chunk = bytes.slice(offset, offset + SPEECH_STREAM_MAX_CHUNK_BYTES);
queue.push(chunk);
queuedBytes += chunk.byteLength;
offset += chunk.byteLength;
}
try {
pump();
} catch {
fail(new Error("Could not send microphone audio."));
}
},
finish: () => {
finishing = true;
try {
pump();
} catch {
fail(new Error("Could not finish live transcription."));
}
return final;
},
cancel: abort,
};
}