// @effect-diagnostics globalTimers:off - Native display handoff has a bounded acknowledgement window.
import {
AmbientLight,
Box3,
CanvasTexture,
DirectionalLight,
Euler,
LinearFilter,
PerspectiveCamera,
PMREMGenerator,
Quaternion,
Scene,
SRGBColorSpace,
Vector3,
WebGLRenderer,
} from "three";
import { RoomEnvironment } from "three/addons/environments/RoomEnvironment.js";
import {
createDuoScene,
duoDisplayKey,
duoFrameMatches,
type DuoPanelId,
type DuoHingeLeaf,
} from "./duoScene.ts";
import { loadDeviceModel } from "./modelScene.ts";
import { createDeviceModelSlot, type DeviceModelSource } from "./model.ts";
import { createDeviceFraming } from "./deviceFraming.ts";
import { createDeviceMotion } from "./deviceMotion.ts";
import { duoViewSnaps, nearestDuoView, type DuoRestFace } from "./duoSnap.ts";
import { createRenderScheduler } from "./renderScheduler.ts";
import type { DeviceScreenSize } from "./stream.ts";
import { duoScreenSettled, type DuoPose } from "./duoControl.ts";
export interface DuoViewer {
readonly setScreen: (screen: DeviceScreenSize | null) => void;
readonly setHingePreview: (angle: number | null, pinch?: boolean) => void;
readonly setInteractionActive: (active: boolean, mode?: "touch" | "orbit") => void;
readonly rejectOrientation: () => void;
readonly frameUpdated: (panel: DuoPanelId, primary?: HTMLCanvasElement) => void;
readonly resize: (width: number, height: number, pixelRatio: number) => void;
readonly screenPoint: (
x: number,
y: number,
captured?: boolean,
) => { x: number; y: number } | null;
readonly orbit: (x: number, y: number) => void;
readonly beginHinge: (x: number, y: number) => boolean;
readonly resetPose: () => void;
readonly cancelInput: () => void;
readonly dispose: () => void;
}
export function createDuoViewer(options: {
canvas: HTMLCanvasElement;
sources: Record<DuoPanelId, HTMLCanvasElement>;
model: DeviceModelSource;
onUnavailable: () => void;
onModelError?: (cause: unknown) => void;
onPanelRequested?: (panel: DuoPanelId) => void;
onOrientationRequested?: (orientation: DeviceScreenSize["orientation"]) => void;
}): DuoViewer {
const surfaces = ([1, 3] as const)
.map((id) => {
const canvas = document.createElement("canvas");
canvas.width = id === 1 ? 784 : 1600;
canvas.height = id === 1 ? 1140 : 1125;
const context = canvas.getContext("2d", { alpha: false });
if (!context) throw new Error("Duo display canvas is unavailable");
context.fillStyle = "#080a10";
context.fillRect(0, 0, canvas.width, canvas.height);
return { id, canvas, context };
})
.map(({ id, canvas, context }) => {
const texture = new CanvasTexture(canvas);
texture.colorSpace = SRGBColorSpace;
texture.minFilter = texture.magFilter = LinearFilter;
texture.generateMipmaps = false;
return { id, canvas, context, texture };
});
const renderer = new WebGLRenderer({
canvas: options.canvas,
alpha: true,
antialias: true,
powerPreference: "low-power",
});
renderer.outputColorSpace = SRGBColorSpace;
const scene = new Scene();
const environment = (() => {
const generator = new PMREMGenerator(renderer);
const room = new RoomEnvironment();
try {
return generator.fromScene(room, 0.04);
} catch (cause) {
for (const surface of surfaces) surface.texture.dispose();
renderer.dispose();
renderer.forceContextLoss();
throw cause;
} finally {
room.dispose();
generator.dispose();
}
})();
scene.environment = environment.texture;
const camera = new PerspectiveCamera(36, 1, 0.1, 150);
const key = new DirectionalLight(0xffffff, 4);
key.position.set(-6, 10, 14);
const fill = new DirectionalLight(0xc7dcff, 2);
fill.position.set(8, -3, -5);
scene.add(new AmbientLight(0xffffff, 2.4), key, fill);
let model: ReturnType<typeof createDuoScene> | null = null;
let screen: DeviceScreenSize | null = null;
let readyKey = "";
let primaryKey = "";
let activationAt = 0;
let disposed = false;
let viewport = { width: 0, height: 0, ratio: 1 };
let buffer = { width: 0, height: 0, ratio: 0 };
let restFace: DuoRestFace = "inside";
let requestedOrientation: DeviceScreenSize["orientation"] | null = null;
let viewOrientation: DeviceScreenSize["orientation"] | null = null;
const pivot = new Vector3();
const targetPivot = new Vector3();
let requestedPanel: DuoPanelId | null = null;
let handoffTimer: ReturnType<typeof setTimeout> | null = null;
const clearHandoff = () => {
if (handoffTimer) clearTimeout(handoffTimer);
handoffTimer = null;
requestedPanel = null;
};
const activeSnaps = () =>
model
? duoViewSnaps(
model.restFrames(screen?.screenId === 1 ? 1 : 3),
screen?.screenId === 1 ? 1 : 3,
)
: [];
const snaps = () => {
if (!model || !screen?.supportsPhysicalOrientation || !options.onPanelRequested)
return activeSnaps();
const cover = duoViewSnaps(model.restFrames(1), 1);
return angle > 20 && angle < 180
? [...duoViewSnaps(model.restFrames(3), 3), ...cover]
: activeSnaps();
};
const orbit = createDeviceMotion({
choose(rotation) {
const snap = nearestDuoView(rotation, snaps());
if (!snap) return rotation;
restFace = snap.face;
const panel = snap.face === "cover" ? 1 : 3;
if (screen && panel !== (requestedPanel ?? screen.screenId) && options.onPanelRequested) {
clearHandoff();
requestedPanel = panel;
model?.cancelInput();
options.onPanelRequested(panel);
handoffTimer = setTimeout(() => {
clearHandoff();
const confirmed = nearestDuoView(orbit.rotation, activeSnaps());
if (confirmed) {
restFace = confirmed.face;
orbit.setPose(confirmed.rotation, performance.now());
}
scheduler.invalidate();
}, 5000);
} else if (
screen &&
requestedPanel === null &&
snap.orientation !== viewOrientation &&
options.onOrientationRequested
) {
viewOrientation = snap.orientation;
requestedOrientation = snap.orientation;
options.onOrientationRequested(snap.orientation);
}
return snap.rotation;
},
});
let angle = 180;
let targetAngle = 180;
let previewAngle: number | null = null;
let hingeLeaf: DuoHingeLeaf | null = null;
let appliedAngle = Number.NaN;
let interactionActive = false;
const framing = createDeviceFraming();
const framingBounds = new Box3();
let firstPose = true;
let foldPending = false;
let physicalPose: DuoPose = "open";
let presentationAngle = 180;
let targetPresentation = new Quaternion();
let lastTime = 0;
const reduced =
typeof matchMedia === "function" && matchMedia("(prefers-reduced-motion: reduce)").matches;
const applyPose = () => {
if (!model) return;
const before = hingeLeaf && appliedAngle !== angle ? model.leafRotation(hingeLeaf) : null;
model.setAngle(angle);
appliedAngle = angle;
if (before && hingeLeaf) {
// Rebase both the displayed rotation and its rest target so releasing a
// pinch cannot resume the pre-fold body rotation.
const correction = before.multiply(model.leafRotation(hingeLeaf).invert());
if (correction.angleTo(new Quaternion()) > 1e-8)
orbit.setPose(orbit.rotation.clone().multiply(correction), performance.now(), true);
}
model.root.quaternion.copy(orbit.rotation);
model.root.position.set(0, 0, 0);
const frames = model.restFrames(restFace === "cover" ? 1 : 3);
const frame = frames.find((frame) => frame.face === restFace) ?? frames[0];
targetPivot.copy(frame?.center ?? new Vector3());
model.root.position.copy(pivot).applyQuaternion(model.root.quaternion).negate();
};
const updateCamera = () => {
camera.position.set(framing.center.x, framing.center.y, framing.distance());
camera.lookAt(framing.center.x, framing.center.y, 0);
camera.updateProjectionMatrix();
};
const fit = (immediate = false) => {
if (!model || !viewport.width || !viewport.height) return;
camera.aspect = viewport.width / viewport.height;
framing.setBounds(
framingBounds.setFromObject(model.root),
(camera.fov * Math.PI) / 360,
camera.aspect,
performance.now(),
immediate,
);
updateCamera();
};
const moving = () =>
Math.abs(angle - targetAngle) > 0.01 || pivot.distanceTo(targetPivot) > 0.001;
const scheduler = createRenderScheduler(() => {
if (disposed || !model || !viewport.width || !viewport.height) return;
try {
const now = performance.now();
const elapsed = Math.max(0, (now - lastTime) / 1000);
const amount = reduced
? 1
: 1 - Math.exp(-14 * Math.max(0, (now - lastTime) / 1000 || 0.016));
lastTime = now;
const inMotion = moving();
const orbitChanged = orbit.advance(now, reduced);
if (inMotion) {
angle += (targetAngle - angle) * amount;
if (Math.abs(angle - targetAngle) <= 0.01) angle = targetAngle;
applyPose();
pivot.lerp(targetPivot, amount);
if (pivot.distanceTo(targetPivot) <= 0.001) pivot.copy(targetPivot);
applyPose();
fit(reduced || elapsed > 0.5);
}
if (orbitChanged) {
applyPose();
fit(reduced || elapsed > 0.5);
}
if (framing.advance(now, reduced)) updateCamera();
if (
buffer.width !== viewport.width ||
buffer.height !== viewport.height ||
buffer.ratio !== viewport.ratio
) {
renderer.setDrawingBufferSize(viewport.width, viewport.height, viewport.ratio);
buffer = viewport;
}
renderer.render(scene, camera);
if (inMotion || moving() || orbit.needsFrame() || framing.needsFrame())
scheduler.invalidate();
} catch {
options.onUnavailable();
}
});
const slot = createDeviceModelSlot({
load: loadDeviceModel,
onError(cause) {
options.onModelError?.(cause);
options.onUnavailable();
},
install(loaded) {
const next = loaded
? createDuoScene(loaded.asset, { 1: surfaces[0]!.texture, 3: surfaces[1]!.texture })
: null;
if (model) {
scene.remove(model.root);
model.dispose();
}
model = next;
appliedAngle = Number.NaN;
if (!model || disposed) return;
scene.add(model.root);
applyPose();
if (screen?.screenId === 1 && physicalPose !== "tent") {
const snap = nearestDuoView(orbit.rotation, activeSnaps());
if (snap) {
restFace = snap.face;
orbit.setPose(snap.rotation, performance.now(), true);
applyPose();
}
}
pivot.copy(targetPivot);
applyPose();
fit();
scheduler.invalidate();
},
});
const lost = (event: Event) => {
event.preventDefault();
options.onUnavailable();
};
options.canvas.addEventListener("webglcontextlost", lost);
slot.set(options.model);
return {
setScreen(next) {
if (disposed) return;
const wasMoving = moving() || orbit.needsFrame();
if (duoDisplayKey(screen) !== duoDisplayKey(next)) {
readyKey = "";
activationAt = performance.now();
model?.cancelInput();
}
const previous = screen;
const ownedHandoff = requestedPanel !== null;
if (!next || next.screenId === requestedPanel) clearHandoff();
const requestedRotation = requestedOrientation !== null;
const ownedRotation = requestedRotation && next?.screenId === previous?.screenId;
const changedDisplay = next?.screenId !== previous?.screenId;
requestedOrientation = null;
screen = next;
const changedPose = next?.hingePose && next.hingePose !== previous?.hingePose;
const folding = hingeLeaf !== null && !changedPose;
const angleBefore = previous?.hingeAngle ?? (previous?.screenId === 1 ? 0 : 180);
const angleAfter = next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180);
if (!next || folding || changedPose) foldPending = false;
else if (previous && angleBefore !== angleAfter) foldPending = true;
const landedFold = foldPending && !!next && duoScreenSettled(next) && !ownedHandoff;
if (next && duoScreenSettled(next)) foldPending = false;
const rotated =
!foldPending &&
next?.screenId === previous?.screenId &&
next?.hingeAngle === previous?.hingeAngle &&
next?.orientation !== previous?.orientation;
const leftPhysicalPose =
rotated &&
!ownedRotation &&
!ownedHandoff &&
!folding &&
!next?.hingePose &&
previewAngle === null;
if (firstPose || changedPose || leftPhysicalPose || landedFold) {
physicalPose = next?.hingePose ?? (next?.screenId === 1 ? "closed" : "open");
presentationAngle = next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180);
}
// A command reply/config confirms hinge state. Display identity, never an angle heuristic, owns input.
targetAngle = previewAngle ?? next?.hingeAngle ?? (next?.screenId === 1 ? 0 : 180);
if (
firstPose ||
changedPose ||
landedFold ||
(rotated && !ownedRotation && !ownedHandoff && !folding)
) {
hingeLeaf = null;
viewOrientation = next?.orientation ?? null;
const poseAngle = presentationAngle;
const fold = ((180 - poseAngle) * Math.PI) / 360;
let roll = physicalPose === "closed" ? 0 : Math.PI / 2;
if (next && next.width < next.height) {
if (next.orientation === "landscape_left") roll -= Math.PI / 2;
if (next.orientation === "landscape_right") roll += Math.PI / 2;
}
if (next?.orientation === "portrait_upside_down") roll -= Math.PI;
const physical =
physicalPose === "laptop"
? new Euler(-fold + Math.PI / 9, -Math.PI / 9, Math.PI / 2, "YXZ")
: physicalPose === "tent"
? new Euler(Math.PI / 2 + Math.PI / 18, -Math.PI / 9, -Math.PI / 2, "YXZ")
: new Euler(0, (Math.PI / 2) * Math.pow(1 - poseAngle / 180, 3), 0, "YXZ");
targetPresentation = new Quaternion().setFromEuler(physical);
if (physicalPose !== "laptop" && physicalPose !== "tent")
targetPresentation.premultiply(
new Quaternion().setFromAxisAngle(new Vector3(0, 0, 1), roll),
);
if (next?.screenId === 1 && !landedFold && physicalPose !== "tent") {
// Open presets can retain native cover readback; face the display's actual hinged normal.
// Landed folds already face the cover while the hinge normal is stale.
// Tent must stand on both edges with the cover outward; facing it flat would tip it over.
const snap = nearestDuoView(targetPresentation, activeSnaps());
if (snap) targetPresentation.copy(snap.rotation);
}
if (firstPose) angle = targetAngle;
restFace = next?.screenId === 1 ? "cover" : physicalPose === "laptop" ? "right" : "inside";
orbit.setPose(targetPresentation, performance.now(), firstPose);
firstPose = false;
} else if (
next &&
changedDisplay &&
!ownedHandoff &&
!folding &&
(restFace === "cover") !== (next.screenId === 1) &&
(requestedRotation || duoScreenSettled(next))
) {
// A sensor rotation can hand ownership to the other native display.
// Face that display without sending another rotation and creating a feedback loop.
// Native election can briefly flap to the display a fold does not show and
// back; neither is a handoff, and the view may still be easing into place.
const snap = nearestDuoView(orbit.rotation, activeSnaps());
if (snap) {
restFace = snap.face;
orbit.setPose(snap.rotation, performance.now());
}
}
if (!wasMoving) lastTime = performance.now();
applyPose();
fit();
scheduler.invalidate();
},
setHingePreview(next, pinch = false) {
if (disposed || (next !== null && (!Number.isFinite(next) || next < 0 || next > 180))) return;
if (!moving()) lastTime = performance.now();
previewAngle = next;
if (next !== null && !pinch) {
hingeLeaf = null;
} else if (next !== null && !hingeLeaf && model) {
clearHandoff();
requestedOrientation = null;
hingeLeaf = screen?.screenId === 1 && angle > 20 ? "left" : "right";
orbit.setPose(orbit.rotation.clone(), performance.now(), true);
}
orbit.hold(interactionActive || (pinch && next !== null), performance.now());
targetAngle = next ?? screen?.hingeAngle ?? (screen?.screenId === 1 ? 0 : 180);
if (next !== null && pinch) {
angle = next;
applyPose();
fit();
}
model?.cancelInput();
scheduler.invalidate();
},
setInteractionActive(active, mode = "touch") {
if (disposed) return;
if (mode === "orbit") {
if (active) hingeLeaf = null;
orbit.dragActive(active, performance.now());
} else {
interactionActive = active;
orbit.hold(active || previewAngle !== null, performance.now());
framing.hold(active, performance.now());
}
scheduler.invalidate();
},
rejectOrientation() {
if (disposed || (requestedOrientation === null && requestedPanel === null)) return;
clearHandoff();
requestedOrientation = null;
viewOrientation = screen?.orientation ?? null;
const confirmed = activeSnaps().filter((snap) => snap.orientation === screen?.orientation);
const snap = nearestDuoView(orbit.rotation, confirmed);
if (snap) {
restFace = snap.face;
orbit.setPose(snap.rotation, performance.now());
}
scheduler.invalidate();
},
frameUpdated(id, primary) {
if (disposed || screen?.screenId !== id) return;
const key = duoDisplayKey(screen);
// The elected native feed is authoritative during handoff. Fixed-panel
// encoders can retain an inactive blank until that surface changes again.
if (!primary && primaryKey === key) return;
const source = primary ?? options.sources[id];
if (!duoFrameMatches(source, screen)) return;
const surface = surfaces[id === 1 ? 0 : 1]!;
const { context, canvas } = surface;
// Ignore native shutdown blanks only while waiting for an activation. Steady black application content remains valid.
if (!primary && !readyKey && performance.now() - activationAt < 1500) {
const probe = document.createElement("canvas");
probe.width = probe.height = 8;
const probeContext = probe.getContext("2d", { willReadFrequently: true });
if (probeContext) {
probeContext.drawImage(source, 0, 0, 8, 8);
if (
!probeContext
.getImageData(0, 0, 8, 8)
.data.some((value, index) => index % 4 !== 3 && value > 3)
)
return;
}
}
context.save();
context.fillStyle = "#080a10";
context.fillRect(0, 0, canvas.width, canvas.height);
context.translate(canvas.width / 2, canvas.height / 2);
// The inner panel is mounted a quarter turn from the native framebuffer.
const rotate = id === 3;
if (rotate) context.rotate(Math.PI / 2);
const scale = Math.min(
canvas.width / (rotate ? source.height : source.width),
canvas.height / (rotate ? source.width : source.height),
);
context.drawImage(
source,
(-source.width * scale) / 2,
(-source.height * scale) / 2,
source.width * scale,
source.height * scale,
);
context.restore();
surface.texture.needsUpdate = true;
readyKey = duoDisplayKey(screen);
if (primary) primaryKey = readyKey;
scheduler.invalidate();
},
resize(width, height, ratio) {
if (disposed || ![width, height, ratio].every(Number.isFinite) || width <= 0 || height <= 0)
return;
viewport = { width, height, ratio: Math.min(2, Math.max(1, ratio)) };
fit(true);
scheduler.invalidate();
},
screenPoint(x, y, captured = false) {
if (
disposed ||
moving() ||
previewAngle !== null ||
requestedOrientation !== null ||
requestedPanel !== null
)
return null;
applyPose();
return model?.screenPoint(x, y, camera, screen, readyKey, captured) ?? null;
},
cancelInput() {
hingeLeaf = null;
model?.cancelInput();
},
orbit(x, y) {
if (disposed) return;
hingeLeaf = null;
orbit.orbit(x * viewport.width, y * viewport.height, performance.now());
scheduler.invalidate();
},
beginHinge(x, y) {
if (disposed || !model || !screen) return false;
applyPose();
const leaf = model.hingeLeafAt(x, y, camera);
if (!leaf) return false;
clearHandoff();
requestedOrientation = null;
hingeLeaf = screen.screenId === 1 && angle > 20 ? "left" : "right";
orbit.setPose(orbit.rotation.clone(), performance.now(), true);
model.cancelInput();
return true;
},
resetPose() {
if (disposed) return;
hingeLeaf = null;
requestedOrientation = null;
clearHandoff();
const snap = activeSnaps().find((snap) => snap.orientation === screen?.orientation);
if (snap && snap.orientation !== viewOrientation && options.onOrientationRequested) {
viewOrientation = snap.orientation;
requestedOrientation = snap.orientation;
options.onOrientationRequested(snap.orientation);
}
orbit.reset(snap?.rotation ?? targetPresentation, performance.now());
applyPose();
fit();
scheduler.invalidate();
},
dispose() {
clearHandoff();
if (disposed) return;
disposed = true;
scheduler.dispose();
slot.dispose();
options.canvas.removeEventListener("webglcontextlost", lost);
for (const surface of surfaces) surface.texture.dispose();
scene.environment = null;
environment.dispose();
renderer.dispose();
renderer.forceContextLoss();
},
};
}