sailfork / t3code-github / apps/ desktop/ src/ snapShot/ SnapShotAccessibility.ts

// @effect-diagnostics globalTimers:off -- Accessibility timeouts run outside Effect fibers.

import {
  SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS,
  type SnapShotAccessibility,
} from "@t3tools/contracts";
import type * as Electron from "electron";

import {
  accessibleWindowElementTree,
  accessibleWindowText,
  compactAccessibilityTree,
  findAccessibleWindow,
  isWaylandSession,
} from "./snapShot.ts";

const ACCESSIBILITY_TIMEOUT_MS = 3_000;

export type AccessibleWindowIdentity = {
  readonly title: string;
  readonly bounds: Electron.Rectangle;
  readonly clientBounds?: Electron.Rectangle;
  readonly owner: { readonly processId: number; readonly bundleId?: string };
  readonly accessibilityBoundsReliable?: boolean;
};

export type CapturedWindowAccessibilityContext = {
  readonly accessibleText?: string;
  readonly accessibility?: SnapShotAccessibility;
};

export type SnapShotAccessibilityRequest = {
  readonly active: AccessibleWindowIdentity;
  readonly platform: NodeJS.Platform;
  readonly sourceTitle: string;
  readonly imageSize: Electron.Size;
};

type AccessibilityApp = (typeof import("@crowecawcaw/xa11y"))["App"];

type AccessibilityReadProgress = {
  accessibleText: string | undefined;
  flatComplete: boolean;
  richComplete: boolean;
  richLocationsReliable: boolean;
  richRoot?: Extract<SnapShotAccessibility, { format: "element-tree" }>["root"];
  richTruncated: boolean;
  timedOut: boolean;
};

function accessibilityReadSnapshot(
  progress: AccessibilityReadProgress,
  imageSize: Electron.Size,
): CapturedWindowAccessibilityContext | undefined {
  const richTree = progress.richRoot
    ? compactAccessibilityTree(progress.richRoot, {
        descendantLocationsReliable: progress.richLocationsReliable,
      })
    : undefined;
  const richText = richTree
    ? accessibleWindowText(richTree.root, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS)
    : undefined;
  const accessibleText = progress.accessibleText ?? richText;
  const richTruncated = progress.richTruncated || richTree?.truncated === true;
  const accessibility: SnapShotAccessibility | undefined =
    progress.richComplete &&
    richTree &&
    (!richTruncated || !progress.accessibleText || progress.accessibleText === richText)
      ? {
          format: "element-tree",
          coordinateSpace: "captured-image",
          imageSize,
          truncated: richTruncated,
          root: richTree.root,
        }
      : progress.flatComplete && progress.accessibleText
        ? {
            format: "flat-text",
            text: progress.accessibleText,
            truncated: progress.accessibleText.length >= SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS,
          }
        : richTree
          ? {
              format: "element-tree",
              coordinateSpace: "captured-image",
              imageSize,
              truncated: true,
              root: richTree.root,
            }
          : undefined;
  if (!accessibleText && !accessibility) return undefined;
  return JSON.parse(
    JSON.stringify({
      ...(accessibleText ? { accessibleText } : {}),
      ...(accessibility ? { accessibility } : {}),
    }),
  ) as CapturedWindowAccessibilityContext;
}

type AccessibilityElement = Awaited<ReturnType<InstanceType<AccessibilityApp>["children"]>>[number];

async function windowsForPid(
  App: AccessibilityApp,
  processId: number,
): Promise<readonly AccessibilityElement[]> {
  return await App.byPid(processId, { timeout: 0 })
    .then((app) => app.children())
    .catch(() => []);
}

/** Flatpak apps reach AT-SPI through xdg-dbus-proxy, so the compositor PID misses. */
async function windowsFromAppList(App: AccessibilityApp): Promise<readonly AccessibilityElement[]> {
  const apps = await App.list().catch(() => []);
  return (await Promise.all(apps.map((app) => app.children().catch(() => [])))).flat();
}

async function readCapturedWindowAccessibility(
  App: AccessibilityApp,
  request: SnapShotAccessibilityRequest,
  progress: AccessibilityReadProgress,
  onStarted: () => void,
): Promise<CapturedWindowAccessibilityContext | undefined> {
  const { active, platform, sourceTitle, imageSize } = request;
  const foreground = platform === "win32" ? await App.foreground({ timeout: 0 }) : undefined;
  const pidWindows =
    foreground !== undefined
      ? foreground.pid === active.owner.processId
        ? [foreground.asElement()]
        : []
      : await windowsForPid(App, active.owner.processId);
  const matchMode = isWaylandSession(platform, process.env) ? "wayland" : "screen-bounds";
  const captured = {
    title: active.title,
    sourceTitle,
    bounds: active.bounds,
    clientBounds: active.clientBounds,
    platform,
    bundleId: pidWindows.length > 0 ? active.owner.bundleId : undefined,
  };
  const window = findAccessibleWindow(
    pidWindows.length > 0 || foreground !== undefined ? pidWindows : await windowsFromAppList(App),
    captured,
    matchMode,
    { allowUntitledUniqueBounds: foreground === undefined && pidWindows.length > 0 },
  );
  if (!window) {
    onStarted();
    return undefined;
  }
  const accessibleBounds = window.bounds;
  const matchingBounds =
    matchMode === "wayland" &&
    active.clientBounds &&
    accessibleBounds &&
    Math.abs(accessibleBounds.width - active.clientBounds.width) <= 2 &&
    Math.abs(accessibleBounds.height - active.clientBounds.height) <= 2
      ? active.clientBounds
      : active.bounds;
  const locationsReliable =
    active.accessibilityBoundsReliable !== false &&
    (matchMode === "screen-bounds" ||
      (accessibleBounds !== null &&
        Math.abs(accessibleBounds.x - matchingBounds.x) <= 2 &&
        Math.abs(accessibleBounds.y - matchingBounds.y) <= 2));
  const flatRead = window
    .tree()
    .then((tree) => {
      progress.accessibleText =
        accessibleWindowText(tree, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS) || undefined;
      progress.flatComplete = true;
    })
    .catch(() => {
      progress.flatComplete = true;
    });
  // Preserve the screenshot frame origin and scale so accessibility coordinates include decorations.
  const sourceBounds =
    matchMode === "wayland" && active.clientBounds
      ? active.bounds
      : (accessibleBounds ?? active.bounds);
  const richRead = accessibleWindowElementTree(window, sourceBounds, imageSize, {
    locationsReliable,
    onProgress: (root, truncated, descendantLocationsReliable) => {
      progress.richLocationsReliable = descendantLocationsReliable;
      progress.richRoot = root;
      progress.richTruncated = truncated;
    },
    shouldContinue: () => !progress.timedOut,
    verifyDescendantLocations: matchMode === "wayland",
  })
    .then((rich) => {
      progress.richComplete = true;
      if (rich) {
        progress.richRoot = rich.root;
        progress.richTruncated = rich.truncated;
      }
    })
    .catch(() => {
      progress.richComplete = true;
    });
  onStarted();
  await Promise.all([flatRead, richRead]);
  return accessibilityReadSnapshot(progress, imageSize);
}

let activeAccessibilityRead: Promise<unknown> | undefined;

async function raceAccessibleRead<T>(
  run: () => Promise<T>,
  timeoutValue: () => T | undefined,
): Promise<T | undefined> {
  if (activeAccessibilityRead) return undefined;
  const read = run().catch(() => undefined);
  activeAccessibilityRead = read;
  void read.finally(() => {
    if (activeAccessibilityRead === read) activeAccessibilityRead = undefined;
  });
  let timeout: ReturnType<typeof setTimeout> | undefined;
  try {
    return await Promise.race([
      read,
      new Promise<T | undefined>((resolve) => {
        timeout = setTimeout(() => resolve(timeoutValue()), ACCESSIBILITY_TIMEOUT_MS);
      }),
    ]);
  } finally {
    if (timeout) clearTimeout(timeout);
  }
}

export function readAccessibleWindowContextWithApp(
  App: AccessibilityApp,
  request: SnapShotAccessibilityRequest,
  onStarted: () => void = () => undefined,
): Promise<CapturedWindowAccessibilityContext | undefined> {
  const progress: AccessibilityReadProgress = {
    accessibleText: undefined,
    flatComplete: false,
    richComplete: false,
    richLocationsReliable: false,
    richTruncated: false,
    timedOut: false,
  };
  return raceAccessibleRead(
    () => readCapturedWindowAccessibility(App, request, progress, onStarted),
    () => {
      progress.timedOut = true;
      return accessibilityReadSnapshot(progress, request.imageSize);
    },
  );
}