sailfork / t3code-github / scripts/ mobile-behavior.ts

#!/usr/bin/env node
// @effect-diagnostics nodeBuiltinImport:off globalDate:off globalTimers:off globalFetch:off - Host-side device automation uses Node subprocess and filesystem APIs directly.

import * as NodeChildProcess from "node:child_process";
import * as NodeFS from "node:fs";
import * as NodeFSP from "node:fs/promises";
import * as NodeOS from "node:os";
import * as NodePath from "node:path";
import * as NodeProcess from "node:process";
import * as NodeReadline from "node:readline";
import * as NodeURL from "node:url";

import { CASES } from "./mobile-behavior/cases.ts";
import { blockingResults, resultsFor, summarize, type SampleRow } from "./mobile-behavior/grade.ts";
import { Device, type Pool, poolError, run } from "./mobile-e2e/device.ts";
import type { Pose } from "./mobile-e2e/poses.ts";

const REPO_ROOT = NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), "..");

/** The devices the behavior cases run on; each runs in parallel with its own server and Metro. */
const TARGETS = [
  { id: "iphone-18-pro", device: "iPhone 18 Pro", form: "phone", behaviorPose: "phone" },
  { id: "ipad-pro-13", device: "iPad Pro 13-inch (M5)", form: "tablet", behaviorPose: "ipad" },
] as const;
type Target = (typeof TARGETS)[number];

const log = (scope: string, message: string) => NodeProcess.stderr.write(`[${scope}] ${message}\n`);

interface TargetRun {
  readonly target: Target;
  readonly directory: string;
  readonly error?: string;
}

/** Keeps a lease alive while the cases run: they go minutes without an agent-device call. */
function renewWhileRunning(pool: Pool, stateDir: string, session: string, token: string) {
  const gateway = pool.agentDeviceUrl.replace(/\/v1\/agent-device\/?$/, "");
  const timer = setInterval(() => {
    const record = NodePath.join(stateDir, "remote-connections", `${session}.json`);
    void NodeFSP.readFile(record, "utf8")
      .then((text) => (JSON.parse(text) as { leaseId: string }).leaseId)
      .then((leaseId) =>
        fetch(`${gateway}/v1/leases/${encodeURIComponent(leaseId)}/renew`, {
          method: "POST",
          headers: { authorization: `Bearer ${token}` },
        }),
      )
      .catch(() => undefined);
  }, 90_000);
  return () => clearInterval(timer);
}

async function runTarget(
  target: Target,
  pool: Pool,
  runId: string,
  outputDirectory: string,
  config: string,
  stateDir: string,
  token: string,
  passThrough: ReadonlyArray<string>,
): Promise<TargetRun> {
  const directory = NodePath.join(outputDirectory, target.id);
  await NodeFSP.mkdir(directory, { recursive: true });
  const pose: Pose = {
    id: target.id,
    platform: "ios",
    device: target.device,
    form: target.form,
    appearance: "light",
  };
  let device: Device | null = null;
  for (let attempt = 1; !device; attempt++) {
    try {
      log(target.id, `leasing ${target.device}`);
      device = await Device.lease(pose, pool, attempt === 1 ? runId : `${runId}-retry`);
      log(target.id, "installing the dev client");
      await device.install();
    } catch (error) {
      const message = error instanceof Error ? error.message : String(error);
      await device?.release();
      device = null;
      if (attempt >= 2 || !poolError(message)) return { target, directory, error: message };
      log(target.id, `retrying after: ${message.split("\n")[0]}`);
    }
  }
  const stopRenewing = renewWhileRunning(pool, stateDir, device.session, token);
  try {
    log(target.id, "running the behavior cases");
    const child = NodeChildProcess.spawn(
      NodeProcess.execPath,
      [
        "scripts/mobile-behavior/run.ts",
        "--out",
        directory,
        "--label",
        target.id,
        "--pose",
        target.behaviorPose,
        "--device-config",
        config,
        "--device-session",
        device.session,
        "--agent-device",
        pool.agentDevice,
        ...passThrough,
      ],
      { cwd: REPO_ROOT, env: NodeProcess.env, stdio: ["ignore", "pipe", "pipe"] },
    );
    const transcript = NodeFS.createWriteStream(NodePath.join(directory, "run.log"));
    // The scoreboard and any fatal error go to stdout, progress to stderr; the transcript keeps both.
    NodeReadline.createInterface({ input: child.stdout }).on("line", (line) =>
      transcript.write(`${line}\n`),
    );
    NodeReadline.createInterface({ input: child.stderr }).on("line", (line) => {
      transcript.write(`${line}\n`);
      if (/#\d+: /.test(line)) log(target.id, line);
    });
    const code = await new Promise<number>((resolve) =>
      child.on("close", (exit) => resolve(exit ?? 1)),
    );
    transcript.end();
    const hasResults = NodeFS.existsSync(NodePath.join(directory, "results.jsonl"));
    return hasResults || code === 0
      ? { target, directory }
      : { target, directory, error: `the cases stopped (exit ${code}); see ${directory}/run.log` };
  } finally {
    stopRenewing();
    await device.release();
  }
}

async function main() {
  const args = NodeProcess.argv.slice(2);
  const values = (flag: string) =>
    args.flatMap((arg, index) => (args[index - 1] === flag ? [arg] : []));
  if (args.includes("--list")) {
    for (const target of TARGETS) NodeProcess.stdout.write(`${target.id}\t${target.device}\n`);
    for (const testCase of CASES)
      NodeProcess.stdout.write(`${testCase.id}\t${testCase.kind}\t${testCase.source}\n`);
    return;
  }
  const selected = values("--device");
  const unknown = selected.filter((id) => !TARGETS.some((target) => target.id === id));
  if (unknown.length) throw new Error(`Unknown device ${unknown.join(", ")}. Run with --list.`);
  const targets = TARGETS.filter((target) => selected.length === 0 || selected.includes(target.id));
  const passThrough = [
    ...values("--samples").flatMap((value) => ["--samples", value]),
    ...values("--case").flatMap((value) => ["--case", value]),
  ];

  const agentDeviceUrl = NodeProcess.env.SIMLOCK_AGENT_DEVICE_URL;
  const token = NodeProcess.env.AGENT_DEVICE_DAEMON_AUTH_TOKEN;
  if (!agentDeviceUrl || !token)
    throw new Error(
      "Set SIMLOCK_AGENT_DEVICE_URL (<gateway>/v1/agent-device) and AGENT_DEVICE_DAEMON_AUTH_TOKEN (a Simlock agent token) to run on the device pool.",
    );

  const runId = new Date().toISOString().replaceAll(/[:.]/g, "-");
  const outputDirectory = NodePath.join(REPO_ROOT, "artifacts/mobile-behavior", runId);
  await NodeFSP.mkdir(outputDirectory, { recursive: true });
  const rootDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-mobile-behavior-"));
  const workspace = await NodeFSP.realpath(outputDirectory);
  const config = NodePath.join(rootDir, "agent-device.json");
  const stateDir = NodePath.join(rootDir, "agent-device");
  await NodeFSP.writeFile(
    config,
    JSON.stringify({ daemonBaseUrl: agentDeviceUrl, daemonAuthToken: token, stateDir }),
    { mode: 0o600 },
  );
  const pool: Pool = {
    repoRoot: REPO_ROOT,
    agentDevice: NodePath.join(REPO_ROOT, "scripts/node_modules/agent-device/bin/agent-device.mjs"),
    agentDeviceUrl,
    workspace,
    env: { ...NodeProcess.env, AGENT_DEVICE_CONFIG: config, PWD: workspace },
  };
  try {
    // A cache miss builds the dev client, which can outlast a lease, so it is ready first.
    log("behavior", "preparing the ios dev client");
    await run(
      NodeProcess.execPath,
      ["scripts/mobile-native-client.ts", "ensure", "ios", "--prepare"],
      { cwd: REPO_ROOT, timeoutMs: 1_800_000 },
    );
    const runs = await Promise.all(
      targets.map((target) =>
        runTarget(target, pool, runId, outputDirectory, config, stateDir, token, passThrough),
      ),
    );

    const rows = runs.flatMap((entry) => {
      const file = NodePath.join(entry.directory, "results.jsonl");
      if (!NodeFS.existsSync(file)) return [];
      return NodeFS.readFileSync(file, "utf8")
        .split("\n")
        .filter(Boolean)
        .map((line) => JSON.parse(line) as SampleRow);
    });
    const results = resultsFor(rows, CASES);
    const stopped = runs.filter((entry) => entry.error);
    const summary = summarize({
      cases: CASES,
      results,
      header: [
        `# Mobile behavior · ${runId}`,
        "",
        "Cells show pass or FAIL, passed/samples, and the key metric's median. `bad` cases reproduce known bugs and are expected to fail until they are fixed.",
        ...stopped.map((entry) => `\n**${entry.target.id} did not finish:** ${entry.error}`),
      ].join("\n"),
    });
    const summaryFile = NodePath.join(outputDirectory, "summary.md");
    await NodeFSP.writeFile(summaryFile, summary);
    const blocking = blockingResults(results, CASES);
    for (const result of blocking)
      log(result.device, `${result.id}: ${result.pass === null ? "incomplete" : "FAIL"}`);
    for (const entry of stopped) log(entry.target.id, entry.error!);
    NodeProcess.stdout.write(
      `${JSON.stringify({ summary: summaryFile, cases: results.length, blocking: blocking.length, stopped: stopped.length })}\n`,
    );
    process.exitCode = blocking.length || stopped.length ? 1 : 0;
  } finally {
    await NodeFSP.rm(rootDir, { recursive: true, force: true });
  }
}

if (import.meta.main) {
  await main();
}