// @effect-diagnostics globalTimers:off -- Accessibility timeouts run outside Effect fibers.
import {
SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS,
type SnapShotAccessibility,
} from "@t3tools/contracts";
import type * as Electron from "electron";
import {
accessibleWindowElementTree,
accessibleWindowText,
compactAccessibilityTree,
findAccessibleWindow,
isWaylandSession,
} from "./snapShot.ts";
const ACCESSIBILITY_TIMEOUT_MS = 3_000;
export type AccessibleWindowIdentity = {
readonly title: string;
readonly bounds: Electron.Rectangle;
readonly clientBounds?: Electron.Rectangle;
readonly owner: { readonly processId: number; readonly bundleId?: string };
readonly accessibilityBoundsReliable?: boolean;
};
export type CapturedWindowAccessibilityContext = {
readonly accessibleText?: string;
readonly accessibility?: SnapShotAccessibility;
};
export type SnapShotAccessibilityRequest = {
readonly active: AccessibleWindowIdentity;
readonly platform: NodeJS.Platform;
readonly sourceTitle: string;
readonly imageSize: Electron.Size;
};
type AccessibilityApp = (typeof import("@crowecawcaw/xa11y"))["App"];
type AccessibilityReadProgress = {
accessibleText: string | undefined;
flatComplete: boolean;
richComplete: boolean;
richLocationsReliable: boolean;
richRoot?: Extract<SnapShotAccessibility, { format: "element-tree" }>["root"];
richTruncated: boolean;
timedOut: boolean;
};
function accessibilityReadSnapshot(
progress: AccessibilityReadProgress,
imageSize: Electron.Size,
): CapturedWindowAccessibilityContext | undefined {
const richTree = progress.richRoot
? compactAccessibilityTree(progress.richRoot, {
descendantLocationsReliable: progress.richLocationsReliable,
})
: undefined;
const richText = richTree
? accessibleWindowText(richTree.root, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS)
: undefined;
const accessibleText = progress.accessibleText ?? richText;
const richTruncated = progress.richTruncated || richTree?.truncated === true;
const accessibility: SnapShotAccessibility | undefined =
progress.richComplete &&
richTree &&
(!richTruncated || !progress.accessibleText || progress.accessibleText === richText)
? {
format: "element-tree",
coordinateSpace: "captured-image",
imageSize,
truncated: richTruncated,
root: richTree.root,
}
: progress.flatComplete && progress.accessibleText
? {
format: "flat-text",
text: progress.accessibleText,
truncated: progress.accessibleText.length >= SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS,
}
: richTree
? {
format: "element-tree",
coordinateSpace: "captured-image",
imageSize,
truncated: true,
root: richTree.root,
}
: undefined;
if (!accessibleText && !accessibility) return undefined;
return JSON.parse(
JSON.stringify({
...(accessibleText ? { accessibleText } : {}),
...(accessibility ? { accessibility } : {}),
}),
) as CapturedWindowAccessibilityContext;
}
type AccessibilityElement = Awaited<ReturnType<InstanceType<AccessibilityApp>["children"]>>[number];
async function windowsForPid(
App: AccessibilityApp,
processId: number,
): Promise<readonly AccessibilityElement[]> {
return await App.byPid(processId, { timeout: 0 })
.then((app) => app.children())
.catch(() => []);
}
/** Flatpak apps reach AT-SPI through xdg-dbus-proxy, so the compositor PID misses. */
async function windowsFromAppList(App: AccessibilityApp): Promise<readonly AccessibilityElement[]> {
const apps = await App.list().catch(() => []);
return (await Promise.all(apps.map((app) => app.children().catch(() => [])))).flat();
}
async function readCapturedWindowAccessibility(
App: AccessibilityApp,
request: SnapShotAccessibilityRequest,
progress: AccessibilityReadProgress,
onStarted: () => void,
): Promise<CapturedWindowAccessibilityContext | undefined> {
const { active, platform, sourceTitle, imageSize } = request;
const foreground = platform === "win32" ? await App.foreground({ timeout: 0 }) : undefined;
const pidWindows =
foreground !== undefined
? foreground.pid === active.owner.processId
? [foreground.asElement()]
: []
: await windowsForPid(App, active.owner.processId);
const matchMode = isWaylandSession(platform, process.env) ? "wayland" : "screen-bounds";
const captured = {
title: active.title,
sourceTitle,
bounds: active.bounds,
clientBounds: active.clientBounds,
platform,
bundleId: pidWindows.length > 0 ? active.owner.bundleId : undefined,
};
const window = findAccessibleWindow(
pidWindows.length > 0 || foreground !== undefined ? pidWindows : await windowsFromAppList(App),
captured,
matchMode,
{ allowUntitledUniqueBounds: foreground === undefined && pidWindows.length > 0 },
);
if (!window) {
onStarted();
return undefined;
}
const accessibleBounds = window.bounds;
const matchingBounds =
matchMode === "wayland" &&
active.clientBounds &&
accessibleBounds &&
Math.abs(accessibleBounds.width - active.clientBounds.width) <= 2 &&
Math.abs(accessibleBounds.height - active.clientBounds.height) <= 2
? active.clientBounds
: active.bounds;
const locationsReliable =
active.accessibilityBoundsReliable !== false &&
(matchMode === "screen-bounds" ||
(accessibleBounds !== null &&
Math.abs(accessibleBounds.x - matchingBounds.x) <= 2 &&
Math.abs(accessibleBounds.y - matchingBounds.y) <= 2));
const flatRead = window
.tree()
.then((tree) => {
progress.accessibleText =
accessibleWindowText(tree, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS) || undefined;
progress.flatComplete = true;
})
.catch(() => {
progress.flatComplete = true;
});
// Preserve the screenshot frame origin and scale so accessibility coordinates include decorations.
const sourceBounds =
matchMode === "wayland" && active.clientBounds
? active.bounds
: (accessibleBounds ?? active.bounds);
const richRead = accessibleWindowElementTree(window, sourceBounds, imageSize, {
locationsReliable,
onProgress: (root, truncated, descendantLocationsReliable) => {
progress.richLocationsReliable = descendantLocationsReliable;
progress.richRoot = root;
progress.richTruncated = truncated;
},
shouldContinue: () => !progress.timedOut,
verifyDescendantLocations: matchMode === "wayland",
})
.then((rich) => {
progress.richComplete = true;
if (rich) {
progress.richRoot = rich.root;
progress.richTruncated = rich.truncated;
}
})
.catch(() => {
progress.richComplete = true;
});
onStarted();
await Promise.all([flatRead, richRead]);
return accessibilityReadSnapshot(progress, imageSize);
}
let activeAccessibilityRead: Promise<unknown> | undefined;
async function raceAccessibleRead<T>(
run: () => Promise<T>,
timeoutValue: () => T | undefined,
): Promise<T | undefined> {
if (activeAccessibilityRead) return undefined;
const read = run().catch(() => undefined);
activeAccessibilityRead = read;
void read.finally(() => {
if (activeAccessibilityRead === read) activeAccessibilityRead = undefined;
});
let timeout: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
read,
new Promise<T | undefined>((resolve) => {
timeout = setTimeout(() => resolve(timeoutValue()), ACCESSIBILITY_TIMEOUT_MS);
}),
]);
} finally {
if (timeout) clearTimeout(timeout);
}
}
export function readAccessibleWindowContextWithApp(
App: AccessibilityApp,
request: SnapShotAccessibilityRequest,
onStarted: () => void = () => undefined,
): Promise<CapturedWindowAccessibilityContext | undefined> {
const progress: AccessibilityReadProgress = {
accessibleText: undefined,
flatComplete: false,
richComplete: false,
richLocationsReliable: false,
richTruncated: false,
timedOut: false,
};
return raceAccessibleRead(
() => readCapturedWindowAccessibility(App, request, progress, onStarted),
() => {
progress.timedOut = true;
return accessibilityReadSnapshot(progress, request.imageSize);
},
);
}