sailfork / t3code-github / packages/ client-runtime/ src/ device/ phoneViewer.ts

import {
  AmbientLight,
  Box3,
  CanvasTexture,
  DirectionalLight,
  LinearFilter,
  Matrix4,
  PerspectiveCamera,
  Quaternion,
  Scene,
  SRGBColorSpace,
  Vector3,
  WebGLRenderer,
} from "three";
import {
  createDeviceModelSlot,
  type DeviceModelSource,
  type DeviceAccessorySource,
} from "./model.ts";
import { createImportedPhoneScene, loadDeviceModel } from "./modelScene.ts";
import { createPhoneScene, phoneDisplayLayout } from "./phoneScene.ts";
import {
  createAndroidFoldScene,
  DEFAULT_FOLD_INNER_ASPECT,
  isFoldInnerAspect,
} from "./androidFoldScene.ts";
import { createRenderScheduler } from "./renderScheduler.ts";
import { createDeviceMotion } from "./deviceMotion.ts";
import { createDeviceFraming } from "./deviceFraming.ts";
import { nearestDeviceView } from "./deviceViewSnap.ts";
import type { DeviceScreenSize } from "./stream.ts";
import { IOS_PHONE_SHAPE, type DeviceShapeProfile } from "./shapeProfile.ts";

export interface PhoneViewer {
  readonly setModel: (source: DeviceModelSource | null) => void;
  readonly setAccessory: (source: DeviceAccessorySource | null) => void;
  readonly frameUpdated: () => void;
  readonly setScreen: (screen: DeviceScreenSize | null, profile?: DeviceShapeProfile) => void;
  readonly setFoldAngle: (angle: number | null) => void;
  readonly resize: (width: number, height: number, pixelRatio: number) => void;
  readonly screenPoint: (
    x: number,
    y: number,
    captured?: boolean,
  ) => { x: number; y: number } | null;
  readonly orbit: (deltaX: number, deltaY: number) => void;
  readonly setInteractionActive: (active: boolean, mode: "touch" | "orbit") => void;
  readonly resetPose: () => void;
  readonly dispose: () => void;
}

const ANDROID_ORIENTATION_TURN_MS = 450;
const ANDROID_FOLD_TURN_MS = 850;

/** Owns only presentation resources. The caller retains the decoded canvas and the stream connection. */
export function createPhoneViewer(options: {
  readonly canvas: HTMLCanvasElement;
  readonly source: HTMLCanvasElement;
  readonly onUnavailable: () => void;
  readonly onModelError?: (cause: unknown) => void;
  readonly onFramingAspect?: (aspect: number) => void;
  readonly profile?: DeviceShapeProfile;
  readonly model?: DeviceModelSource | null;
  readonly accessory?: DeviceAccessorySource | null;
  readonly foldAngle?: number | null;
}): PhoneViewer {
  const renderer = new WebGLRenderer({
    canvas: options.canvas,
    alpha: true,
    antialias: true,
    powerPreference: "low-power",
  });
  renderer.outputColorSpace = SRGBColorSpace;
  const makeTexture = () => {
    const next = new CanvasTexture(options.source);
    next.colorSpace = SRGBColorSpace;
    next.minFilter = LinearFilter;
    next.magFilter = LinearFilter;
    next.generateMipmaps = false;
    return next;
  };
  let texture = makeTexture();
  let textureWidth = options.source.width;
  let textureHeight = options.source.height;
  const scene = new Scene();
  const camera = new PerspectiveCamera(32, 1, 0.1, 30);
  camera.position.z = 5.5;
  const ambient = new AmbientLight(0xffffff, 2.4);
  const key = new DirectionalLight(0xe4edff, 5);
  key.position.set(-3, 4, 5);
  const rim = new DirectionalLight(0xffffff, 4);
  rim.position.set(3, 1, -3);
  const fill = new DirectionalLight(0x9facd4, 2);
  fill.position.set(-2, -2, -4);
  scene.add(ambient, key, rim, fill);

  let screen: DeviceScreenSize | null = null;
  let layout = phoneDisplayLayout(screen, options.source.width, options.source.height);
  let profile = options.profile ?? IOS_PHONE_SHAPE;
  let foldAngle = options.foldAngle ?? null;
  let orientationAngle =
    foldAngle !== null && profile.id.startsWith("android") ? 0 : layout.rotation;
  let orientationTurn: { from: number; to: number; startedAt: number } | null = null;
  let imported: Awaited<ReturnType<typeof loadDeviceModel>> | null = null;
  let modelSource = options.model ?? null;
  let accessory: Awaited<ReturnType<typeof loadDeviceModel>> | null = null;
  let accessoryBounds: Box3 | null = null;
  let foldTurn: { from: number; to: number; startedAt: number } | null = null;
  const rawAspect = () => options.source.width / options.source.height;
  let foldAspect = isFoldInnerAspect(rawAspect()) ? rawAspect() : DEFAULT_FOLD_INNER_ASPECT;
  const createFoldScene = (angle: number, displayLayout = layout) =>
    createAndroidFoldScene(texture, displayLayout, angle, foldAspect);
  const visibleFoldAngle = (fallback: number) => {
    if (!foldTurn) return fallback;
    const progress = Math.min(1, (performance.now() - foldTurn.startedAt) / ANDROID_FOLD_TURN_MS);
    const eased = progress * progress * (3 - 2 * progress);
    return foldTurn.from + (foldTurn.to - foldTurn.from) * eased;
  };
  let phone: ReturnType<typeof createPhoneScene> | ReturnType<typeof createAndroidFoldScene> =
    foldAngle !== null && profile.id.startsWith("android")
      ? createFoldScene(foldAngle)
      : createPhoneScene(texture, layout, profile);
  scene.add(phone.root);
  let disposed = false;
  const rest = new Quaternion();
  const motion = createDeviceMotion({
    choose: (rotation) =>
      nearestDeviceView(rotation, [{ rotation: new Quaternion(), yawLimit: Math.PI / 3 }])!
        .rotation,
  });
  motion.setPose(rest, performance.now(), true);
  const framing = createDeviceFraming();
  const reducedMotion = () =>
    globalThis.matchMedia?.("(prefers-reduced-motion: reduce)").matches ?? false;
  let viewport = { width: 0, height: 0, pixelRatio: 1 };
  let drawingBuffer = { width: 0, height: 0, pixelRatio: 0 };

  let framingAspect: number | null = null;
  const fit = (immediate = false) => {
    if (!viewport.width || !viewport.height) return;
    camera.aspect = viewport.width / viewport.height;
    const bounds = new Box3(
      new Vector3(-phone.width / 2, -phone.height / 2, 0),
      new Vector3(phone.width / 2, phone.height / 2, 0),
    );
    if (imported && accessoryBounds) bounds.union(accessoryBounds);
    bounds.applyMatrix4(new Matrix4().makeRotationZ(orientationAngle));
    const size = bounds.getSize(new Vector3());
    const aspect = size.x / size.y;
    if (aspect !== framingAspect) {
      framingAspect = aspect;
      options.onFramingAspect?.(aspect);
    }
    phone.root.updateMatrixWorld(true);
    framing.setBounds(
      new Box3().setFromObject(phone.root),
      (camera.fov * Math.PI) / 360,
      camera.aspect,
      performance.now(),
      immediate,
    );
    applyCamera();
  };
  const applyCamera = () => {
    camera.position.set(framing.center.x, framing.center.y, framing.distance());
    camera.lookAt(framing.center.x, framing.center.y, 0);
    camera.updateProjectionMatrix();
  };
  const applyPose = () => {
    phone.root.quaternion.copy(motion.rotation);
    phone.orientation.rotation.z = orientationAngle;
  };
  const scheduler = createRenderScheduler(() => {
    if (disposed || !viewport.width || !viewport.height) return;
    try {
      if (
        drawingBuffer.width !== viewport.width ||
        drawingBuffer.height !== viewport.height ||
        drawingBuffer.pixelRatio !== viewport.pixelRatio
      ) {
        // Canvas allocation clears the previous image. Commit it with the redraw,
        // rather than exposing an empty buffer between ResizeObserver and the next frame.
        renderer.setDrawingBufferSize(viewport.width, viewport.height, viewport.pixelRatio);
        drawingBuffer = viewport;
      }
      const now = performance.now();
      if (motion.advance(now, reducedMotion())) {
        applyPose();
        fit(reducedMotion());
      }
      if (orientationTurn) {
        const progress = Math.min(
          1,
          (now - orientationTurn.startedAt) / ANDROID_ORIENTATION_TURN_MS,
        );
        const eased = progress * progress * (3 - 2 * progress);
        orientationAngle =
          orientationTurn.from + (orientationTurn.to - orientationTurn.from) * eased;
        if (progress === 1) orientationTurn = null;
        applyPose();
        fit(reducedMotion());
      }
      if (foldTurn && "setAngle" in phone) {
        const progress = Math.min(1, (now - foldTurn.startedAt) / ANDROID_FOLD_TURN_MS);
        const eased = progress * progress * (3 - 2 * progress);
        phone.setAngle(foldTurn.from + (foldTurn.to - foldTurn.from) * eased);
        if (progress === 1) foldTurn = null;
        fit(reducedMotion());
      }
      framing.advance(now, reducedMotion());
      applyCamera();
      renderer.render(scene, camera);
      if (motion.needsFrame() || framing.needsFrame() || orientationTurn || foldTurn)
        scheduler.invalidate();
    } catch {
      options.onUnavailable();
    }
  });
  const updateLayout = (nextProfile = profile) => {
    const next = phoneDisplayLayout(screen, options.source.width, options.source.height);
    const resized =
      textureWidth !== options.source.width || textureHeight !== options.source.height;
    if (
      resized ||
      nextProfile !== profile ||
      next.aspect !== layout.aspect ||
      next.rawLandscape !== layout.rawLandscape ||
      next.rotation !== layout.rotation
    ) {
      if (resized) {
        const previous = texture;
        texture = makeTexture();
        textureWidth = options.source.width;
        textureHeight = options.source.height;
        phone.setDisplay(texture, next);
        previous.dispose();
      }
      const frameAspect = rawAspect();
      const innerChanged = isFoldInnerAspect(frameAspect) && frameAspect !== foldAspect;
      if (innerChanged) foldAspect = frameAspect;
      if (!imported && "setAngle" in phone && innerChanged) {
        const angle = visibleFoldAngle(foldAngle ?? 180);
        scene.remove(phone.root);
        phone.dispose();
        phone = createFoldScene(angle, next);
        scene.add(phone.root);
      } else if (
        !imported &&
        !("setAngle" in phone) &&
        (nextProfile !== profile || next.aspect !== layout.aspect)
      ) {
        scene.remove(phone.root);
        phone.dispose();
        phone = createPhoneScene(texture, next, nextProfile);
        scene.add(phone.root);
      } else {
        phone.setDisplay(texture, next);
      }
      if (next.rotation !== layout.rotation) {
        if (nextProfile.id.startsWith("android") && !("setAngle" in phone) && !reducedMotion()) {
          const difference = Math.atan2(
            Math.sin(next.rotation - orientationAngle),
            Math.cos(next.rotation - orientationAngle),
          );
          orientationTurn = {
            from: orientationAngle,
            to: orientationAngle + difference,
            startedAt: performance.now(),
          };
        } else {
          orientationTurn = null;
          orientationAngle = "setAngle" in phone ? 0 : next.rotation;
        }
      }
      layout = next;
      profile = nextProfile;
      applyPose();
      fit(!orientationTurn);
    }
    applyPose();
  };
  const modelSlot = createDeviceModelSlot({
    load: loadDeviceModel,
    onError: options.onModelError,
    install(model) {
      const next = disposed
        ? null
        : model
          ? createImportedPhoneScene(model.asset, texture, layout)
          : foldAngle !== null && profile.id.startsWith("android")
            ? createFoldScene(foldAngle)
            : createPhoneScene(texture, layout, profile);
      foldTurn = null;
      scene.remove(phone.root);
      phone.dispose();
      imported = model;
      if (!next) return;
      phone = next;
      scene.add(phone.root);
      if (accessory) {
        if (imported) phone.orientation.add(accessory.asset);
        else accessory.asset.removeFromParent();
      }
      applyPose();
      fit(true);
      scheduler.invalidate();
    },
  });
  const accessorySlot = createDeviceModelSlot({
    load: loadDeviceModel,
    onError: options.onModelError,
    install(model) {
      const bounds = model ? new Box3().setFromObject(model.asset) : null;
      if (
        bounds &&
        (bounds.isEmpty() ||
          ![...bounds.min.toArray(), ...bounds.max.toArray()].every(Number.isFinite))
      )
        throw new Error("Device accessory has invalid bounds");
      accessory?.asset.removeFromParent();
      accessory = model;
      accessoryBounds = bounds;
      if (disposed) return;
      if (imported && accessory) phone.orientation.add(accessory.asset);
      fit(true);
      scheduler.invalidate();
    },
  });
  const setAccessory = (source: DeviceAccessorySource | null) => {
    accessorySlot.set(source?.modelId === modelSource?.id ? source : null);
  };
  const contextLost = (event: Event) => {
    event.preventDefault();
    options.onUnavailable();
  };
  options.canvas.addEventListener("webglcontextlost", contextLost);
  applyPose();
  modelSlot.set(options.model ?? null);
  setAccessory(options.accessory ?? null);
  return {
    setModel(source) {
      if (source?.id !== modelSource?.id || source?.url !== modelSource?.url)
        accessorySlot.set(null);
      modelSource = source;
      modelSlot.set(source);
    },
    setAccessory,
    setFoldAngle(next) {
      if (disposed || next === foldAngle) return;
      const previous = foldAngle;
      foldAngle = next;
      if (imported) return;
      if (next === null || !("setAngle" in phone)) {
        scene.remove(phone.root);
        phone.dispose();
        phone = next === null ? createPhoneScene(texture, layout, profile) : createFoldScene(next);
        scene.add(phone.root);
        orientationTurn = null;
        orientationAngle = next === null ? layout.rotation : 0;
        foldTurn = null;
        applyPose();
        fit(true);
      } else {
        const from = visibleFoldAngle(previous ?? next);
        if (reducedMotion()) {
          foldTurn = null;
          phone.setAngle(next);
          fit(true);
        } else {
          foldTurn = { from, to: next, startedAt: performance.now() };
        }
      }
      scheduler.invalidate();
    },
    frameUpdated() {
      if (disposed) return;
      updateLayout();
      texture.needsUpdate = true;
      scheduler.invalidate();
    },
    setScreen(next, nextProfile = profile) {
      if (disposed) return;
      screen = next;
      updateLayout(nextProfile);
      scheduler.invalidate();
    },
    resize(width, height, pixelRatio) {
      if (disposed) return;
      if (![width, height, pixelRatio].every(Number.isFinite) || width <= 0 || height <= 0) return;
      const ratio = Math.min(2, Math.max(1, pixelRatio));
      if (viewport.width === width && viewport.height === height && viewport.pixelRatio === ratio)
        return;
      viewport = { width, height, pixelRatio: ratio };
      fit(true);
      scheduler.invalidate();
    },
    screenPoint(x, y, captured = false) {
      if (disposed) return null;
      applyPose();
      return phone.screenPoint(x, y, camera, captured);
    },
    orbit(deltaX, deltaY) {
      if (disposed) return;
      motion.orbit(deltaX * viewport.width, deltaY * viewport.height, performance.now());
      scheduler.invalidate();
    },
    setInteractionActive(active, mode) {
      if (disposed) return;
      const now = performance.now();
      if (mode === "orbit") motion.dragActive(active, now);
      else {
        motion.hold(active, now);
        framing.hold(active, now);
      }
      scheduler.invalidate();
    },
    resetPose() {
      if (disposed) return;
      motion.reset(rest, performance.now());
      scheduler.invalidate();
    },
    dispose() {
      if (disposed) return;
      disposed = true;
      scheduler.dispose();
      const hadImported = imported !== null;
      accessorySlot.dispose();
      modelSlot.dispose();
      options.canvas.removeEventListener("webglcontextlost", contextLost);
      scene.remove(phone.root);
      if (!hadImported) phone.dispose();
      texture.dispose();
      renderer.dispose();
      renderer.forceContextLoss();
    },
  };
}