import {
createPreviewFramePainter,
createPreviewStreamClient,
type PreviewStreamClient,
} from "@t3tools/client-runtime/preview/server-browser-stream";
import type { EnvironmentId, ThreadId } from "@t3tools/contracts";
import { scopedThreadKey } from "@t3tools/client-runtime/environment";
import { previewPdfAssetUrl } from "@t3tools/shared/preview";
import { useSyncExternalStore } from "react";
import { readPreviewStreamAccess } from "~/state/previewStream";
import { previewStateAtom, type ThreadPreviewState } from "~/previewStateStore";
import { appAtomRegistry } from "~/rpc/atomRegistry";
interface WebKitVideo {
webkitSetPresentationMode?: (mode: string) => void;
webkitPresentationMode?: string;
}
interface ActivePictureInPicture {
readonly key: string;
readonly stop: () => void;
}
// Owns its stream outside React so the floating window survives panel unmounts.
let active: ActivePictureInPicture | null = null;
const listeners = new Set<() => void>();
const emit = () => {
for (const listener of listeners) listener();
};
const subscribe = (listener: () => void) => {
listeners.add(listener);
return () => listeners.delete(listener);
};
export const serverPictureInPictureKey = (threadId: string, tabId: string) =>
JSON.stringify([threadId, tabId]);
export function supportsServerPictureInPicture(): boolean {
if (typeof document === "undefined") return false;
if (typeof HTMLCanvasElement.prototype.captureStream !== "function") return false;
const video = HTMLVideoElement.prototype as HTMLVideoElement & WebKitVideo;
return (
(document.pictureInPictureEnabled && typeof video.requestPictureInPicture === "function") ||
typeof video.webkitSetPresentationMode === "function"
);
}
export function useServerPictureInPictureKey(): string | null {
return useSyncExternalStore(
subscribe,
() => active?.key ?? null,
() => null,
);
}
export function closeServerPictureInPicture(): void {
const current = active;
active = null;
current?.stop();
emit();
}
/** Open during user activation, seeding the video with the currently visible frame. */
export async function openServerPictureInPicture(input: {
readonly environmentId: EnvironmentId;
readonly threadId: ThreadId;
readonly tabId: string;
readonly seed: HTMLCanvasElement | null;
}): Promise<void> {
closeServerPictureInPicture();
const canvas = document.createElement("canvas");
const context = canvas.getContext("2d");
const seed = input.seed && input.seed.width > 0 && input.seed.height > 0 ? input.seed : null;
canvas.width = seed?.width ?? 640;
canvas.height = seed?.height ?? 400;
// The capture emits a frame per draw, so the seed is drawn after it starts.
const stream = canvas.captureStream();
if (seed) context?.drawImage(seed, 0, 0);
else context?.fillRect(0, 0, canvas.width, canvas.height);
const video = document.createElement("video") as HTMLVideoElement & WebKitVideo;
video.muted = true;
video.playsInline = true;
video.srcObject = stream;
// Safari only floats a video that is in the document.
video.style.cssText =
"position:fixed;right:0;bottom:0;width:1px;height:1px;opacity:0;pointer-events:none";
document.body.append(video);
const painter = createPreviewFramePainter(canvas);
let client: PreviewStreamClient | null = null;
let stopped = false;
let refusals = 0;
let retryTimer: ReturnType<typeof setTimeout> | null = null;
let stopWatchingPreview = () => {};
const onLeave = () => {
if (active === current) closeServerPictureInPicture();
};
const connect = async (refresh: boolean) => {
const access = await readPreviewStreamAccess(input.environmentId, refresh);
if (stopped) return;
if (!access) {
closeServerPictureInPicture();
return;
}
client = createPreviewStreamClient(
{
access,
threadId: input.threadId,
tabId: input.tabId,
maxWidth: 1280,
maxHeight: 1280,
interactive: false,
},
{
onFrame: (jpeg) => {
refusals = 0;
painter.paint(jpeg);
},
onViewport: () => undefined,
onConnectedChange: () => undefined,
onUnauthorized: () => {
if (stopped) return;
if (refusals >= 5) {
onLeave();
return;
}
retryTimer = setTimeout(() => void connect(true), 1_000 * 2 ** refusals++);
},
onGone: onLeave,
},
);
};
const onPresentationMode = () => {
if (video.webkitPresentationMode !== "picture-in-picture") onLeave();
};
const current: ActivePictureInPicture = {
key: serverPictureInPictureKey(input.threadId, input.tabId),
stop: () => {
stopped = true;
stopWatchingPreview();
if (retryTimer !== null) clearTimeout(retryTimer);
client?.stop();
painter.stop();
video.removeEventListener("leavepictureinpicture", onLeave);
video.removeEventListener("webkitpresentationmodechanged", onPresentationMode);
if (document.pictureInPictureElement === video) {
void document.exitPictureInPicture().catch(() => undefined);
} else if (video.webkitPresentationMode === "picture-in-picture") {
video.webkitSetPresentationMode?.("inline");
}
for (const track of stream.getTracks()) track.stop();
video.srcObject = null;
video.remove();
},
};
active = current;
emit();
try {
await video.play();
if (video.readyState < HTMLMediaElement.HAVE_METADATA) {
await new Promise((resolve) =>
video.addEventListener("loadedmetadata", resolve, { once: true }),
);
}
if (document.pictureInPictureEnabled && typeof video.requestPictureInPicture === "function") {
video.addEventListener("leavepictureinpicture", onLeave);
await video.requestPictureInPicture();
} else {
video.addEventListener("webkitpresentationmodechanged", onPresentationMode);
video.webkitSetPresentationMode?.("picture-in-picture");
}
} catch (error) {
onLeave();
throw error;
}
if (active === current) {
const atom = previewStateAtom(scopedThreadKey(input));
const closeForPdf = (state: ThreadPreviewState) => {
const navigation = state.sessions[input.tabId]?.navStatus;
if (
navigation?._tag === "Success" &&
previewPdfAssetUrl(navigation.url, navigation.url) !== null
) {
onLeave();
}
};
// PDFs render in the client and cannot appear in the server's video stream.
stopWatchingPreview = appAtomRegistry.subscribe(atom, closeForPdf);
closeForPdf(appAtomRegistry.get(atom));
if (active === current) void connect(false);
}
}