sailfork / t3code-github / evals/ mcp-tool-use/ device-grading.ts

import * as Schema from "effect/Schema";

export const DeviceExpect = Schema.Struct({
  briefs: Schema.optional(Schema.Number),
  recording: Schema.optional(Schema.String),
  artifact: Schema.optional(Schema.String),
  noRawUi: Schema.optional(Schema.Boolean),
  capture: Schema.optional(
    Schema.Struct({
      hostId: Schema.String,
      deviceId: Schema.String,
      app: Schema.optional(Schema.String),
    }),
  ),
  noShutdown: Schema.optional(
    Schema.Array(Schema.Struct({ hostId: Schema.String, deviceId: Schema.String })),
  ),
  report: Schema.optional(Schema.Array(Schema.String)),
  reportOrCapture: Schema.optional(Schema.Boolean),
  maxFailedAttemptsPerTurn: Schema.optional(Schema.Number),
  maxCaptureAttempts: Schema.optional(Schema.Number),
});

type DeviceRun = {
  readonly launchers?: ReadonlyArray<string>;
  readonly text?: string;
  readonly finalText?: string;
  readonly stopReason?: string;
  readonly transcript?: ReadonlyArray<unknown>;
  readonly calls: ReadonlyArray<{
    readonly step: number;
    readonly tool: string;
    readonly input: unknown;
    readonly turn?: number;
    readonly failed?: boolean;
    readonly output?: string;
  }>;
};
const record = (value: unknown): value is Readonly<Record<string, unknown>> =>
  typeof value === "object" && value !== null && !Array.isArray(value);
const inputString = (input: unknown, key: string) =>
  record(input) && typeof input[key] === "string" ? input[key] : "";
const resultTarget = (call: DeviceRun["calls"][number]) => {
  try {
    const device: unknown = JSON.parse(call.output ?? "").device;
    if (record(device) && typeof device.hostId === "string" && typeof device.id === "string")
      return { hostId: device.hostId, deviceId: device.id };
  } catch {}
};
const inputTarget = (call: DeviceRun["calls"][number]) => ({
  hostId: inputString(call.input, "hostId") || "local",
  deviceId: inputString(call.input, "deviceId"),
});
const openTargets = (previous: DeviceRun["calls"]) => {
  const sessions = new Map<string, ReturnType<typeof inputTarget>>();
  for (const prior of previous) {
    if (prior.failed !== false) continue;
    if (prior.tool === "device_open") {
      const target = resultTarget(prior) ?? inputTarget(prior);
      const key = `${target.hostId}/${target.deviceId}`;
      sessions.delete(key);
      sessions.set(key, target);
    } else if (prior.tool === "device_close") {
      const host = inputString(prior.input, "hostId"),
        id = inputString(prior.input, "deviceId");
      for (const [key, target] of sessions)
        if ((!host || target.hostId === host) && (!id || target.deviceId === id))
          sessions.delete(key);
    }
  }
  return [...sessions.values()];
};
const flagValue = (words: ReadonlyArray<string>, flag: string) => {
  for (const [i, word] of words.entries()) {
    if (i === 0) continue;
    if (word === flag) return words[i + 1];
    if (word.startsWith(`${flag}=`)) return word.slice(flag.length + 1);
  }
};
/** Each successful device_open's launcher, with the device and session it targets. */
const launcherOpens = (previous: DeviceRun["calls"]) =>
  previous.flatMap((call) => {
    if (call.tool !== "device_open" || call.failed !== false) return [];
    try {
      const { device, agentDevice }: Record<string, unknown> = JSON.parse(call.output ?? "");
      if (
        !record(device) ||
        !record(agentDevice) ||
        typeof device.hostId !== "string" ||
        typeof device.id !== "string" ||
        typeof agentDevice.command !== "string"
      )
        return [];
      const targetArgs = Array.isArray(agentDevice.targetArgs)
        ? agentDevice.targetArgs.filter((arg): arg is string => typeof arg === "string")
        : [];
      const session =
        typeof agentDevice.session === "string"
          ? agentDevice.session
          : flagValue(["", ...targetArgs], "--session");
      return [
        { command: agentDevice.command, hostId: device.hostId, deviceId: device.id, session },
      ];
    } catch {
      return [];
    }
  });
/**
 * The device a launcher command drives: the one its --udid, --serial or --session names among
 * the devices its launcher was returned for, else its launcher's only device. Without launcher
 * results, only the sole open device.
 */
const launcherTarget = (words: ReadonlyArray<string>, previous: DeviceRun["calls"]) => {
  const id = flagValue(words, "--udid") ?? flagValue(words, "--serial");
  const session = flagValue(words, "--session");
  const launched = launcherOpens(previous).filter((open) => open.command === words[0]);
  const candidates = (
    launched.length > 0
      ? launched
      : openTargets(previous).map((t) => ({ ...t, session: undefined }))
  ).filter(
    (t) =>
      (id === undefined || t.deviceId === id) &&
      (session === undefined || t.session === undefined || t.session === session),
  );
  return new Set(candidates.map((t) => `${t.hostId}/${t.deviceId}`)).size === 1
    ? candidates.at(-1)
    : undefined;
};
const screenshotTarget = (call: DeviceRun["calls"][number], previous: DeviceRun["calls"]) => {
  const result = resultTarget(call);
  if (result) return result;
  if (call.failed && /open in this thread/.test(call.output ?? "")) return undefined;
  const host = inputString(call.input, "hostId"),
    id = inputString(call.input, "deviceId");
  if (host && id) return inputTarget(call);
  const matches = openTargets(previous).filter(
    (t) => (!host || t.hostId === host) && (!id || t.deviceId === id),
  );
  return id ? matches.at(-1) : matches.length === 1 ? matches[0] : undefined;
};
const managed =
  /device_(?:open|screenshot|action)\b|device (?:panel|manager|tools)|returned\s+(?:(?:bound|pinned|agent-device|cli)\s+)*(?:launcher|driver|command)/;
const ui =
  /\b(?:ui|screens?|screenshots?|screen captures?|record(?:ing|video)?s?|settings|taps?|typ(?:e|ing)|launch(?:es|ing)?|relaunch(?:es|ing)?|deep links?|swipes?|simulator (?:control|actions)|device actions|(?:drive|control|inspect)\s+(?:the\s+)?(?:simulator|device|app))\b/;
const rawCommand =
  /\bsimctl\s+(?:launch|terminate|openurl|ui|privacy|location|push)\b|\bsimctl\s+io\s+(?:[^\s;]+\s+)?(?:screenshot|recordvideo)\b|\bidb\b[^.;]*\b(?:ui|screenshot|launch|video|record)\b|\badb\b[^.;]*\b(?:input|screencap|screenrecord)\b|\blab-(?:shot|ui|launch|capture(?:-set)?)\b|\bmac_agent_device\b/;

export function briefRoute(text: string) {
  const positive = text
    .toLowerCase()
    .replaceAll("’", "'")
    .replace(/[`*]/g, "")
    // "SSH for the non-UI operations" restricts SSH rather than routing UI through it.
    .replace(/\bnon-(?:ui|device|simulator|screen)\b/g, "auxiliary")
    // A trailing prohibition negates its subject, not an earlier independent predicate.
    .replace(
      /(^|[.;\n]\s*)((?:-\s*)?(?:use|capture|take|record|run|call|drive)\b[^.;,\n]*?)\s+and\s+(?=[^.;,\n]*?\b(?:is|are)\s+(?:strictly\s+)?(?:forbidden|prohibited|disallowed|banned|not allowed|not permitted)\b)/g,
      "$1$2. ",
    )
    .replace(
      /\b(?!(?:because|since|as|if|unless|when|so|but)\b)(?:(?!\b(?:because|since|as|if|unless|when|so|but)\b)[^.;,\n])*?\b(?:is|are)\s+(?:strictly\s+)?(?:forbidden|prohibited|disallowed|banned|not allowed|not permitted)\b/g,
      "",
    )
    .replace(/\b(?:do not|don't)\s+forget\s+to\b/g, "")
    .replace(/\b(?:do not|don't)\s+avoid\b/g, "use")
    .replace(/\b(?:should|must)\s+not\s+be\s+avoided\b/g, "is allowed")
    .replace(/\b(?:there is )?no need to avoid\b/g, "use")
    .replace(/\bnot just\b/g, "")
    .replace(/\bbut (?:also )?for\b/g, "and for")
    .replace(/\bavoid limiting (ssh|simctl|adb|idb) to [^.;]+;\s*use it\b/g, "use $1")
    .replace(/(^|[.!;\n]\s*)(for [^.;,\n]+),\s*(?=(?:use|run)\b)/g, "$1$2 ")
    .split(
      /[\n.!;]+(?=\s|$)|\b(?:but|however|while)\b|,\s*(?=(?:screenshots?|recordings?)\s+(?:via|through|go))|(?:,\s*|\band\s+|\bthen\s+)(?=(?:use|run|call)\b)/,
    )
    .map((clause) =>
      (
        clause.split(
          /\b(?:not|never|no|don't|cannot|can't|avoid|forbidden|prohibited|disallowed|rather than|instead of|without)\b/,
        )[0] ?? ""
      )
        .replace(
          /\b(?:ui|screen|settings|simulator|device|app)\s+(logs?|builds?|test runners?)\b/g,
          "$1",
        )
        .replace(/\b(?:record|capture|stream)\s+(?:the\s+)?logs\b/g, "logs")
        .replace(/\b(?:logs? from|reading)\s+screenshot\s+(?:tests?|logs?)\b/g, "logs")
        .replace(
          /\b(?:copy(?:ing)?|fetch(?:ing)?|transfer(?:ring)?|download(?:ing)?|sync(?:ing)?)\s+(?:(?:the|already|saved)\s+)*(?:screenshots?|recordings?|artifacts?)\b/g,
          "artifact transfer",
        ),
    );
  const hasManaged = positive.some((clause) => managed.test(clause));
  const hasRaw = positive.some((clause) => {
    if (rawCommand.test(clause)) return true;
    const routes = clause.matchAll(/\b(?:ssh|simctl|idb|adb|lab scripts?)\b/g);
    for (const route of routes) {
      const before = clause.slice(0, route.index);
      const after = clause.slice(route.index + route[0].length).split(managed)[0] ?? "";
      // Copying already-saved artifacts is auxiliary; keep later capture verbs visible.
      const copies =
        /^\s+(?:to\s+)?(?:copy|fetch|transfer|download|sync)\b/.test(after) ||
        /\b(?:copy|fetch|transfer|download|sync)\s+(?:(?:the|saved)\s+)*(?:screenshots?|recordings?|artifacts?)\s+(?:over|via|through|using)\s*$/.test(
          before,
        );
      if (copies && !/\b(?:take|capture|inspect|record|drive|control)\b/.test(after)) continue;
      const purpose =
        /^\s+(?:(?:is|are|can|may|must|should|be|used|only|allowed|permitted|fine)\s+)*(?:for|to|into)\b/.test(
          after,
        );
      const hostPurpose =
        /^\s+(?:-\S+\s+\S+\s+)*\S+\s+(?:for|and|then)\b|^\s+(?:handles|and|then)\b/.test(after);
      if ((purpose || hostPurpose || new RegExp(`^\\s+${ui.source}`).test(after)) && ui.test(after))
        return true;
      const via = new RegExp(
        `${ui.source}(?:\\s+(?:may|can|must|should|be|is|are|taken|captured|saved|recorded|only|directly))*\\s+(?:via|by|over|through|using|with|use|run)\\s+(?:raw\\s+)?

	
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
	evals/mcp-tool-use/device-grading.ts at main · sailfork/t3code-github · Sailfork
	
		
, ); if ( via.test(before) || /\b(?:screenshots?|screen(?: capture)?|recordings?)(?:\s+on\s+\S+)?\s+(?:over|via|using|through|with)\s*$/.test( before, ) || /\bfor\s+[^.;]*\b(?:screenshots?|recordings?|screen capture)\s*,?\s*$/.test(before) ) return true; } return false; }); return hasManaged && hasRaw ? "mixed" : hasManaged ? "managed" : hasRaw ? "raw" : "neutral"; } const executable = (word = "") => word.split("/").at(-1) ?? ""; const uiCommand = /^(?:simctl\s+(?:launch|terminate|openurl|ui|privacy|location|push)\b|simctl\s+io\s+\S+\s+(?:screenshot|recordvideo)\b|idb\b.*\b(?:ui|screenshot|launch|video|record)\b|adb\b.*\b(?:shell\s+(?:input|am\s+start|settings)|(?:shell|exec-out)\s+(?:screencap|screenrecord))\b|screencapture\b|lab-(?:shot|ui|launch|capture(?:-set)?)\b)/i; const deviceArgs = (words: ReadonlyArray<string>) => { const args: string[] = []; for (let i = 1; i < words.length; i++) { const word = words[i]!; if (word === "--") return [...args, ...words.slice(i + 1)]; if (!word.startsWith("-")) args.push(word); else if ( /^--(?:platform|udid|serial|device|target|level|config|remote-config|session|state-dir|scope|timeout|timeout-ms|wait|metro-port|out|quality)$/.test( word, ) ) i++; } return args; }; const auxiliaryAgent = /^(?:(?:devices|apps|doctor|capabilities|install|uninstall|reinstall|install-from-source|logs|help|version)(?: |$)|session list(?: |$)|record contact-sheet(?: |$))/; const deviceOperation = (words: ReadonlyArray<string>) => { const args = deviceArgs(words); return !!args[0] && !auxiliaryAgent.test(args.join(" ")); }; const interpreter = /^(?:python\d*(?:\.\d+)?|node|ruby|perl|osascript)$/; const shells = new Set(["bash", "sh", "zsh"]); /** Human and JSON agent-device failure envelopes. */ const AGENT_DEVICE_ERROR = /\bError \([A-Z_]+\):|"success"\s*:\s*false/; export type ShellInvocation = { readonly words: ReadonlyArray<string>; readonly remote: boolean; readonly raw: boolean; readonly managed: boolean; /** Raw-route checks inspect possible actions; managed credit requires established execution. */ readonly executed: boolean; /** An aggregate success cannot establish success of an action masked by ||. */ readonly unmasked: boolean; /** A later `;` or `&` command owns the call's exit status. */ readonly sequenced: boolean; }; // A deliberately bounded shell grammar, not a shell executor. Keep quotes until expansion so // literal command examples, comments and quoted heredocs never become invocations. const tokens = (source: string) => source .replace(/'[^']*'|\\[\s\S]/g, (part) => (part === "\\\n" ? "" : part)) .match( /`(?:\\.|[^`])*`|[<>]\((?:[^()]|\([^()]*\))*\)|\$\((?:[^()'"\\]|\\.|'[^']*'|"(?:\\.|[^"\\])*"|\([^()]*\))*\)|\d+(?:>>?|<)(?:&\d+)?|(?:"(?:\\.|[^"\\])*"|'[^']*'|\\.|\$\{\w+\}|[^\s;&|(){}<>"'\\])+|&&|\|\||[;\n&|(){}]|\d*(?:>>?|<)(?:&\d+)?/g, ) ?? []; const unquote = (word: string, variables: Map<string, string>) => word.replace( /'([^']*)'|"((?:\\.|[^"\\])*)"|\\(.)|([^'"\\]+)/g, ( _part, single: string | undefined, double: string | undefined, escaped: string | undefined, plain: string | undefined, ) => { if (single !== undefined) return single; if (escaped !== undefined) return escaped; return (double ?? plain ?? "").replace( /\\(["\\$])|\$(?:\{(\w+)\}|(\w+))/g, (ref, escaped: string | undefined, braced: string | undefined, bare: string | undefined) => escaped ?? variables.get(braced ?? bare ?? "") ?? ref, ); }, ); function unwrap(words: string[]) { while (words.length) { if (/^\w+=/.test(words[0]!)) { words.shift(); continue; } const name = executable(words[0]); if (!/^(?:env|sudo|command|nohup|exec|time|caffeinate|timeout|nice)$/.test(name)) break; words.shift(); const takesValue: Record<string, RegExp> = { env: /^(?:-u|--unset|-C|--chdir)$/, sudo: /^(?:-[ughpCT]|--user|--group|--host|--prompt)$/, caffeinate: /^-[tw]$/, timeout: /^(?:-[sk]|--signal|--kill-after)$/, nice: /^(?:-n|--adjustment)$/, }; while (words[0]?.startsWith("-")) { const option = words.shift()!; if (option === "--") break; if (takesValue[name]?.test(option)) words.shift(); } if (name === "timeout") words.shift(); } return words; } /** Literal process-launch APIs only. Definitions and strings are not executed programs. */ function scriptInvocations( script: string, remote: boolean, variables: Map<string, string>, language: string, ): ShellInvocation[] { const actions: ShellInvocation[] = []; const python = language.startsWith("python"); // A function body is source, not an invocation. Calls to computed/script functions are // outside this grammar; do not grant managed credit for them. let definitionIndent: number | undefined; if (python) script = script .split("\n") .map((line) => { const indent = line.match(/^\s*/)?.[0].length ?? 0; if (line.trim() && definitionIndent !== undefined && indent <= definitionIndent) definitionIndent = undefined; if (/^\s*(?:async\s+)?def\s/.test(line)) definitionIndent = indent; return definitionIndent === undefined ? line : ""; }) .join("\n"); const launch = /^(subprocess\.(?:run|Popen|call|check_call|check_output)|os\.(?:system|popen)|(?:child_process\.)?(?:execFileSync|execFile|execSync|exec|spawnSync|spawn))\s*\(/; const literal = "(?:'[^']*'|\"(?:\\\\.|[^\"\\\\])*\")"; const quoteArg = (arg: string) => `'${unquote(arg, variables).replaceAll("'", "'\\''")}'`; const array = (source: string) => { const match = new RegExp(`^\\[\\s*(${literal}(?:\\s*,\\s*${literal})*\\s*,?)?\\s*\\]`).exec( source, ); return match ? (match[1]?.match(new RegExp(literal, "g")) ?? []).map(quoteArg).join(" ") : undefined; }; for (let i = 0; i < script.length; i++) { const char = script[i]; if (char === "#" || script.slice(i, i + 2) === "//") { const end = script.indexOf("\n", i); i = end < 0 ? script.length : end; continue; } if (char === "'" || char === '"' || char === "`") { const quote = python && script.slice(i, i + 3) === char.repeat(3) ? char.repeat(3) : char; i += quote.length; while (i < script.length && script.slice(i, i + quote.length) !== quote) { if (script[i] === "\\") i++; i++; } i += quote.length - 1; continue; } const match = launch.exec(script.slice(i)); if (!match) continue; const rest = script.slice(i + match[0].length).trimStart(); let program: string | undefined; if (rest.startsWith("[")) program = array(rest); else { const arg = rest.match(new RegExp(`^${literal}`))?.[0]; const tail = arg === undefined ? "" : rest.slice(arg.length); if (arg && /(?:spawn|execFile)/.test(match[1]!)) { const argv = array(tail.replace(/^\s*,\s*/, "")); if (argv !== undefined) program = `${quoteArg(arg)} ${argv}`; } else if ( arg && (!python || match[1]!.startsWith("os.") || /^\s*,[^)]*\bshell\s*=\s*True\b/.test(tail)) ) { program = unquote(arg, variables); } } if (program !== undefined) { const line = script.slice(script.lastIndexOf("\n", i) + 1, i); const boundary = Math.max( script.lastIndexOf(";", i), python ? script.lastIndexOf("\n", i) : -1, ); const statement = script.slice(boundary + 1, i); const certain = !/=>|\bfunction\b|\blambda\b/.test(statement) && (python ? !/^\s/.test(line) : !script.slice(0, i).includes("{")); actions.push( ...shellInvocations(program, new Map(variables), remote).map((action) => ({ ...action, executed: action.executed && certain, // A script's exit status needn't reflect its subprocesses (`subprocess.run` doesn't check). sequenced: true, })), ); } i += match[0].length - 1; } return actions; } /** Shared by raw-UI, recording and failure grading. Literal aliases may span shell calls. */ export function shellInvocations( command: string, variables = new Map<string, string>(), remote = false, succeeded = false, /** Shared with function calls: the shell's functions, and whether it exited. */ shell: { functions: Map<string, { text: string; certain: boolean }>; exited: boolean; depth: number; } = { functions: new Map(), exited: false, depth: 0, }, ): ShellInvocation[] { if (shell.depth > 8) return []; const bodies = new Map<string, { text: string; expand: boolean }>(); command = command.replace( /^([^\n]*?)<<(-?)\s*(['"]?)(\w+)\3([^\n]*)\n([\s\S]*?)(?:\n[ \t]*\4[ \t]*(?=\n|(?![\s\S]))|(?![\s\S]))/gm, ( _match, line: string, strip: string, quote: string, _end: string, rest: string, body: string, ) => { const key = `__HEREDOC_${bodies.size}`; bodies.set(key, { text: strip ? body.replace(/^\t+/gm, "") : body, expand: !quote }); return `${line} ${key} ${rest}`; }, ); const actions: ShellInvocation[] = []; let clause: string[] = []; let pipe: string | undefined; let gate = ""; let status: boolean | undefined; // `before` says whether an earlier if/elif branch of this statement ran: true, false or unknown. const conditions: Array<{ parent: boolean; certain: boolean; take: boolean | undefined; before: boolean | undefined; }> = []; let active = true; let certain = true; let currentCertain = true; let currentUnmasked = true; let currentSequenced = false; // After a certain `exit` or `return`, nothing later in its scope runs. let exited = false; const append = (nested: ShellInvocation[]) => actions.push( ...nested.map((action) => ({ ...action, executed: action.executed && currentCertain, unmasked: action.unmasked && currentUnmasked, sequenced: action.sequenced || currentSequenced, })), ); const truth = (words: string[]): boolean | undefined => { if (words[0] === "!") { const value = truth(words.slice(1)); return value === undefined ? undefined : !value; } return ["true", ":"].includes(words[0] ?? "") ? true : words[0] === "false" ? false : undefined; }; const commandTokens = tokens(command.replace(/(^|\n)[ \t]*#[^\n]*/g, "$1")); let tokenIndex = 0; const functions = shell.functions; const listTokens = () => { let start = tokenIndex - 1, end = tokenIndex; while (start >= 0 && !/^[;\n&]$/.test(commandTokens[start]!)) start--; while (end < commandTokens.length && !/^[;\n&]$/.test(commandTokens[end]!)) end++; return commandTokens.slice(start + 1, end); }; const substitutions = (source: string, heredoc = false) => { const expanded = heredoc ? source : source.replace( /"((?:\\.|[^"\\])*)"|'[^']*'|\\./g, (part, double: string | undefined) => double ?? (part.startsWith("'") ? "" : part), ); for (const match of expanded.matchAll(/(?<!\\)\$\(((?:[^()]|\([^()]*\))*)\)/g)) append(shellInvocations(match[1]!, new Map(variables), remote)); for (const match of expanded.matchAll(/(?<!\\)`([^`]*)`|[<>]\(((?:[^()]|\([^()]*\))*)\)/g)) append(shellInvocations(match[1] ?? match[2]!, new Map(variables), remote)); }; const flush = () => { if (!clause.length) return; const rawWords = clause; const redirects = redirectOperands; redirectOperands = []; clause = []; let words = rawWords.map((word) => unquote(word, variables)); if (words[0] === "if") { const parent = active; conditions.push({ parent, certain, take: undefined, before: false, }); words.shift(); } if (words[0] === "then") { const condition = conditions.at(-1); // The branch follows the condition list's final status (`if true && false`), not its // first command's. if (condition) condition.take = condition.before === true ? false : status; active = condition?.parent !== false && condition?.take !== false && condition?.before !== true; certain = condition?.certain !== false && condition?.take === true && condition?.before === false; words.shift(); } if (words[0] === "else") { const condition = conditions.at(-1); active = !!condition?.parent && condition.take !== true && condition.before !== true; certain = !!condition?.certain && condition.take === false && condition.before === false; words.shift(); } if (words[0] === "fi") { const condition = conditions.pop(); active = condition?.parent ?? true; certain = condition?.certain ?? true; words.shift(); } if (words[0] === "elif") { // An elif's condition runs only when no earlier branch did. const condition = conditions.at(-1); if (condition) { condition.before = condition.before === true || condition.take === true ? true : condition.before === false && condition.take === false ? false : undefined; condition.take = condition.before === true ? false : truth(words.slice(1)); active = condition.parent && condition.before !== true; certain = condition.certain && condition.before === false; } words.shift(); } const loopList = /^(?:for|select)$/.test(words[0] ?? ""); if (/^(?:while|until)$/.test(words[0] ?? "") || loopList) { const value = loopList ? words[2] === "in" && words.length > 3 ? true : undefined : truth(words.slice(1)); conditions.push({ parent: active, certain, take: words[0] === "until" && value !== undefined ? !value : value, before: false, }); if (!loopList) words.shift(); } if (words[0] === "do") { const condition = conditions.at(-1); active = condition?.parent !== false && condition?.take !== false; certain = condition?.certain !== false && condition?.take === true; words.shift(); } if (words[0] === "done") { const condition = conditions.pop(); active = condition?.parent ?? true; certain = condition?.certain ?? true; words.shift(); } const reachable = gate === "&&" ? status : gate === "||" ? (status === undefined ? undefined : !status) : true; if (!active || exited || !words.length || reachable === false) return; const list = listTokens(); currentUnmasked = !list.includes("||") && words[0] !== "!"; // The call's exit status is the last command's; an action followed by `;` or `&` doesn't own it. // A command piped into another doesn't own the pipeline's status either. currentSequenced = commandTokens[tokenIndex] === "|" || commandTokens.slice(tokenIndex).some((t) => /^[;\n&]$/.test(t)); const finalAndSuccess = succeeded && gate === "&&" && currentUnmasked && !commandTokens.some((t) => /^(?:if|for|while|until|then|else|fi|do|done)$/.test(t)) && !commandTokens.slice(tokenIndex).some((t) => /^[;\n&]$/.test(t)); currentCertain = certain && (reachable === true || finalAndSuccess); // A list's status is known only when this command certainly ran, or when its own result // decides the list either way (`x && false`, `x || true`). const value = truth(words); status = reachable === true ? value : gate === "&&" && value === false ? false : gate === "||" && value === true ? true : undefined; for (const word of [...rawWords, ...redirects]) if (!bodies.has(word)) substitutions(word); if (loopList) return; const body = words.map((word) => bodies.get(word)).find((value) => value !== undefined); if (body?.expand) substitutions(body.text, true); words = words.filter((word) => !bodies.has(word)); const literalAssignments = words.every((word) => /^\w+=/.test(word)); if (literalAssignments || words[0] === "export") { for (const word of words) { const at = word.indexOf("="); if (at > 0) { if (currentCertain) variables.set(word.slice(0, at), word.slice(at + 1)); else variables.delete(word.slice(0, at)); } } return; } if (executable(words[0]) === "command" && /^-[vV]$/.test(words[1] ?? "")) return; while (words[0] === "!") words.shift(); words = unwrap(words); if (/^(?:exit|return)$/.test(executable(words[0]))) { if (currentCertain) { exited = true; // `exit` in a function ends the shell; `return` ends only the function. if (executable(words[0]) === "exit") shell.exited = true; } else certain = false; return; } const name = executable(words[0]); let input = body?.text ?? pipe; pipe = undefined; if (name === "cat" || name === "tee") { pipe = input; return; } if (name === "printf" || name === "echo") { pipe = name === "echo" ? words.slice(1).join(" ") : words[1]?.includes("%s") ? words.slice(2).join("\n") : words[1]?.replaceAll("\\n", "\n"); return; } if (functions.has(name)) { const definition = functions.get(name)!; const call = { functions, exited: false, depth: shell.depth + 1 }; append( shellInvocations(definition.text, new Map(variables), remote, succeeded, call).map( (action) => ({ ...action, executed: action.executed && definition.certain }), ), ); if (call.exited && definition.certain && currentCertain) exited = shell.exited = true; return; } if (name === "ssh") { let at = 1; while (words[at]?.startsWith("-")) { const option = words[at++]!; if (option === "--") break; if (/^-[bBcDEeFIiJLlmOoPpQRSWw]$/.test(option)) at++; } at++; // destination if (words[at] === "--") at++; const script = words.slice(at).join(" "); // A remote shell/interpreter inherits redirected/piped stdin, never the local aliases. const destination = unwrap(tokens(script).map((word) => unquote(word, new Map()))); const reader = executable(destination[0]); // Shells take a program with -c (`bash -se` reads stdin); interpreters with -c or -e. const programFlag = shells.has(reader) ? /^-[a-z]*c[a-z]*$/ : /^-[a-z]*[ce][a-z]*$/; const hasProgram = destination.some((word) => programFlag.test(word)); if ( input !== undefined && (!reader || ((shells.has(reader) || interpreter.test(reader)) && !hasProgram)) ) { if (interpreter.test(reader)) append(scriptInvocations(input, true, new Map(), reader)); else append(shellInvocations(input, new Map(), true)); } append(shellInvocations(script, new Map(), true)); return; } if (shells.has(name) || interpreter.test(name)) { const at = words.findIndex((word) => (shells.has(name) ? /^-[a-z]*c[a-z]*$/ : /^-[a-z]*[ce][a-z]*$/).test(word), ); if (at >= 0) input = words[at + 1]; if (input !== undefined) append( shells.has(name) ? shellInvocations(input, new Map(variables), remote, succeeded) : scriptInvocations(input, remote, variables, name), ); return; } const endOptions = words.indexOf("--"); if ( words .slice(0, endOptions < 0 ? words.length : endOptions) .some((word) => word === "--help" || word === "-h") || words[1] === "help" ) return; const normalized = name === "xcrun" ? words.slice(1) : [...words]; if (name === "xcrun") while (normalized[0]?.startsWith("-")) { const option = normalized.shift(); if (/^(?:--sdk|-sdk|--toolchain|-toolchain)$/.test(option ?? "")) normalized.shift(); } const cli = executable(normalized[0]); const agent = ["agent-device", "mac_agent_device"].includes(cli) && deviceOperation(normalized); actions.push({ words, remote, executed: currentCertain, unmasked: currentUnmasked, sequenced: currentSequenced, raw: uiCommand.test([cli, ...normalized.slice(1)].join(" ")) || (agent && (remote || cli === "mac_agent_device")), managed: agent && !remote && cli === "agent-device", }); }; const scopes: Array<{ variables: Map<string, string>; active: boolean; certain: boolean; exited: boolean; subshell: boolean; }> = []; let comment = false; let redirect = false; let redirectOperands: string[] = []; for (; tokenIndex < commandTokens.length; tokenIndex++) { const token = commandTokens[tokenIndex]!; // Comments come first, so a commented-out function definition is never registered. if (token === "\n") comment = false; if (comment) continue; if (token.startsWith("#")) { comment = true; continue; } // Shell function definitions don't execute their bodies. Keep literal calls supported. const nameAt = tokenIndex + (token === "function" ? 1 : 0); const bodyAt = commandTokens[nameAt + 1] === "(" && commandTokens[nameAt + 2] === ")" ? nameAt + 3 : token === "function" ? nameAt + 1 : -1; if (commandTokens[bodyAt] === "{") { flush(); let end = bodyAt + 1, depth = 1; for (; end < commandTokens.length && depth; end++) { if (commandTokens[end] === "{") depth++; if (commandTokens[end] === "}") depth--; } if (active) functions.set(commandTokens[nameAt]!, { text: commandTokens.slice(bodyAt + 1, end - 1).join(" "), certain, }); tokenIndex = end - 1; continue; } if (/^\d*(?:>>?|<)(?:&\d+)?$/.test(token)) { redirect = !token.includes("&"); continue; } if (redirect) { redirect = false; redirectOperands.push(token); continue; } if (/^(?:&&|\|\||[;\n&|(){}])$/.test(token)) { flush(); if (token === "(" || token === "{") { scopes.push({ variables, active, certain, exited, subshell: token === "(" }); if (token === "(") variables = new Map(variables); const reachable = gate === "&&" ? status : gate === "||" ? status === undefined ? undefined : !status : true; active = active && reachable !== false; certain = certain && reachable === true; } if (token === ")" || token === "}") { const scope = scopes.pop(); // A subshell's exit ends only the subshell; a group's ends the shell. if (scope) exited = scope.exited || (!scope.subshell && exited); if (scope) ({ variables, active, certain } = scope); } if (token !== "|") pipe = undefined; gate = token; } else clause.push(token); } flush(); return actions; } export function gradeDevice(expect: typeof DeviceExpect.Type | undefined, run: DeviceRun) { const checks: Array<{ check: string; pass: boolean }> = []; if (!expect) return checks; const add = (check: string, pass: boolean) => checks.push({ check, pass }); if (expect.briefs !== undefined) { const briefs = run.calls.filter((call) => ["delegate_task", "t3_thread_launch"].includes(call.tool), ); add("device briefs count", briefs.length >= expect.briefs); for (const [i, call] of briefs.entries()) { const route = briefRoute( inputString(call.input, call.tool === "delegate_task" ? "task" : "message"), ); add(`device brief ${i + 1} ${call.tool} ${route}`, route === "managed"); } } if (expect.artifact !== undefined) add( `device final artifact ${expect.artifact}`, run.stopReason === "end_turn" && (run.finalText ?? "").includes(expect.artifact), ); const variables = new Map<string, string>(); const shell = run.calls .filter((call) => ["Bash", "Monitor"].includes(call.tool)) .map( (call) => [ call, shellInvocations( inputString(call.input, "command"), variables, false, call.failed === false, ).map((action) => ({ ...action, args: deviceArgs(action.words), bound: !action.remote && (run.launchers ?? []).includes(action.words[0] ?? ""), })), ] as const, ); const allActions = shell.flatMap(([, actions]) => actions); const usableImage = (call: DeviceRun["calls"][number]) => { if (call.failed !== false) return false; try { const image: unknown = JSON.parse(call.output ?? "").screenshot; return ( record(image) && typeof image.width === "number" && image.width > 0 && typeof image.height === "number" && image.height > 0 ); } catch { return false; } }; const successfulScreenshots = ([call, actions]: (typeof shell)[number]) => { const captures = actions.filter((a) => a.managed && a.bound && a.args[0] === "screenshot"); if ( call.failed !== false || AGENT_DEVICE_ERROR.test(call.output ?? "") || captures.length !== 1 || /"(?:width|height)"\s*:\s*0\b/.test(call.output ?? "") ) return []; const paths = (call.output ?? "").split("\n").flatMap((line) => { // The simulated shell words the success line several ways; each must still name the path. const saved = line.match( /^[^\w\n]*(?:Screenshot saved|Saved screenshot|Captured screenshot|Wrote screenshot)(?: to)?:?\s+(.+)$/i, )?.[1]; if (saved) return [saved]; try { const image: unknown = JSON.parse(line); if ( record(image) && typeof image.path === "string" && typeof image.width === "number" && image.width > 0 && typeof image.height === "number" && image.height > 0 ) return [image.path]; } catch {} return []; }); return captures.filter( (a) => a.unmasked && ((paths.length === 0 && a.executed && !a.sequenced) || paths.some( (path) => path === a.args[1] || (a.args[1]?.startsWith("$/") && path.endsWith(a.args[1].slice(1))), )), ); }; const recovered = expect.reportOrCapture && (run.calls.some((call) => call.tool === "device_screenshot" && usableImage(call)) || shell.some((entry) => successfulScreenshots(entry).length > 0)); if (expect.noShutdown) add( "callsMustNot shutdown occupied device", !run.calls.some( (call, index) => call.tool === "device_close" && record(call.input) && call.input.shutdown === true && openTargets(run.calls.slice(0, index)).some( (target) => expect.noShutdown!.some( (protectedTarget) => target.hostId === protectedTarget.hostId && target.deviceId === protectedTarget.deviceId, ) && (!inputString(call.input, "hostId") || inputString(call.input, "hostId") === target.hostId) && (!inputString(call.input, "deviceId") || inputString(call.input, "deviceId") === target.deviceId), ), ), ); for (const pattern of expect.report ?? []) add( `device final report /${pattern}/`, run.stopReason === "end_turn" && (!!recovered || new RegExp(pattern).test(run.finalText ?? "")), ); if (expect.noRawUi) add("shellMustNot device UI", !allActions.some((action) => action.raw)); if (expect.maxCaptureAttempts !== undefined) add( `maxCalls device captures <= ${expect.maxCaptureAttempts}`, run.calls.filter((call) => call.tool === "device_screenshot").length + allActions.filter((action) => action.managed && action.args[0] === "screenshot").length <= expect.maxCaptureAttempts, ); if (expect.capture) { const target = expect.capture; const matches = (value: ReturnType<typeof inputTarget> | undefined) => value?.hostId === target.hostId && value?.deviceId === target.deviceId; const appSessions = new Set<string>(); let opened = false, captured = false; for (const [index, call] of run.calls.entries()) { if (call.tool === "device_open" && call.failed === false && matches(inputTarget(call))) opened = true; const appReady = !target.app || (call.failed === false && appSessions.size > 0); if ( opened && appReady && call.tool === "device_screenshot" && matches(screenshotTarget(call, run.calls.slice(0, index))) && (!target.app || usableImage(call)) ) captured = true; const entry = shell.find(([source]) => source === call); const actions = (entry?.[1] ?? []).filter((action) => action.managed && action.bound); const screenshots = entry ? successfulScreenshots(entry) : []; for (const action of actions) { const { args, words } = action; const onTarget = matches(launcherTarget(words, run.calls.slice(0, index))); // Only an open or close that ran and succeeded changes the app session. const ran = call.failed === false && action.executed && action.unmasked && !AGENT_DEVICE_ERROR.test(call.output ?? ""); if (target.app && ["open", "close"].includes(args[0] ?? "") && ran) { appSessions.delete(words[0]!); if ( args[0] === "open" && args[1] === target.app && onTarget && (!action.sequenced || (actions.filter((a) => a.args[0] === "open").length === 1 && /^Opened:/m.test(call.output ?? ""))) ) appSessions.add(words[0]!); } if ( opened && onTarget && (target.app ? appSessions.has(words[0]!) && screenshots.includes(action) : action.executed && (["screenshot", "snapshot"].includes(args[0] ?? "") || (args[0] === "record" && args[1] === "start"))) ) captured = true; } } add("device successful open and capture attempt", captured); } if (expect.recording !== undefined) { const steps = [...shell].flatMap(([call, actions]) => { const operations = actions.filter( ({ args }) => args[0] === "record" && /^(?:start|stop)$/.test(args[1] ?? ""), ); return operations .filter( (action) => action.executed && action.unmasked && action.bound && ((!action.sequenced && call.failed === false && !AGENT_DEVICE_ERROR.test(call.output ?? "")) || (operations.filter((o) => o.args[1] === action.args[1]).length === 1 && new RegExp( `(?:^|\\n)[^\\w\\n]*Recording ${action.args[1] === "start" ? "started" : "stopped"}\\b|"kind"\\s*:\\s*"recording${action.args[1] === "start" ? "Started" : "Stopped"}"`, "i", ).test(call.output ?? ""))), ) .map(({ args }) => args[1]!); }); const start = steps.indexOf("start"); checks.push({ check: `device recording ${expect.recording}`, pass: start >= 0 && steps.indexOf("stop", start + 1) >= 0 && run.stopReason === "end_turn" && (run.finalText ?? "").includes(expect.recording), }); } if (expect.maxFailedAttemptsPerTurn !== undefined) { const failedByTurn = new Map<number, number>(); for (const call of run.calls) { const actions = shell.find(([source]) => source === call)?.[1] ?? []; // A failed raw attempt, such as agent-device over ssh, is still a failed device attempt. const device = actions.filter((action) => action.managed || action.raw).length; // agent-device prints one error per failed command, even when a later command succeeds. const errors = (call.output ?? "").match(new RegExp(AGENT_DEVICE_ERROR, "g"))?.length ?? 0; let discoveryFailed = false; if (call.tool === "device_list") { try { const statuses: unknown = JSON.parse(call.output ?? "").hostStatuses; discoveryFailed = record(statuses) && Object.values(statuses).some((s) => record(s) && s.status === "failed"); } catch {} } const attempts = call.tool === "device_list" ? Number(!!call.failed || discoveryFailed) : /^device_(?:open|screenshot|close)$/.test(call.tool) ? Number(!!call.failed) : call.failed && device ? Math.max(1, Math.min(errors, device)) : Math.min(errors, device); const turn = call.turn ?? 0; failedByTurn.set(turn, (failedByTurn.get(turn) ?? 0) + attempts); } const max = expect.maxFailedAttemptsPerTurn; add( `maxCalls device failures per user turn <= ${max}`, [...failedByTurn.values()].every((count) => count <= max), ); } return checks; }
evals/mcp-tool-use/device-grading.ts at main · sailfork/t3code-github · Sailfork
,\n );\n if (\n via.test(before) ||\n /\\b(?:screenshots?|screen(?: capture)?|recordings?)(?:\\s+on\\s+\\S+)?\\s+(?:over|via|using|through|with)\\s*$/.test(\n before,\n ) ||\n /\\bfor\\s+[^.;]*\\b(?:screenshots?|recordings?|screen capture)\\s*,?\\s*$/.test(before)\n )\n return true;\n }\n return false;\n });\n return hasManaged && hasRaw ? \"mixed\" : hasManaged ? \"managed\" : hasRaw ? \"raw\" : \"neutral\";\n}\n\nconst executable = (word = \"\") => word.split(\"/\").at(-1) ?? \"\";\nconst uiCommand =\n /^(?:simctl\\s+(?:launch|terminate|openurl|ui|privacy|location|push)\\b|simctl\\s+io\\s+\\S+\\s+(?:screenshot|recordvideo)\\b|idb\\b.*\\b(?:ui|screenshot|launch|video|record)\\b|adb\\b.*\\b(?:shell\\s+(?:input|am\\s+start|settings)|(?:shell|exec-out)\\s+(?:screencap|screenrecord))\\b|screencapture\\b|lab-(?:shot|ui|launch|capture(?:-set)?)\\b)/i;\nconst deviceArgs = (words: ReadonlyArray\u003cstring>) => {\n const args: string[] = [];\n for (let i = 1; i \u003c words.length; i++) {\n const word = words[i]!;\n if (word === \"--\") return [...args, ...words.slice(i + 1)];\n if (!word.startsWith(\"-\")) args.push(word);\n else if (\n /^--(?:platform|udid|serial|device|target|level|config|remote-config|session|state-dir|scope|timeout|timeout-ms|wait|metro-port|out|quality)$/.test(\n word,\n )\n )\n i++;\n }\n return args;\n};\nconst auxiliaryAgent =\n /^(?:(?:devices|apps|doctor|capabilities|install|uninstall|reinstall|install-from-source|logs|help|version)(?: |$)|session list(?: |$)|record contact-sheet(?: |$))/;\nconst deviceOperation = (words: ReadonlyArray\u003cstring>) => {\n const args = deviceArgs(words);\n return !!args[0] && !auxiliaryAgent.test(args.join(\" \"));\n};\nconst interpreter = /^(?:python\\d*(?:\\.\\d+)?|node|ruby|perl|osascript)$/;\nconst shells = new Set([\"bash\", \"sh\", \"zsh\"]);\n/** Human and JSON agent-device failure envelopes. */\nconst AGENT_DEVICE_ERROR = /\\bError \\([A-Z_]+\\):|\"success\"\\s*:\\s*false/;\n\nexport type ShellInvocation = {\n readonly words: ReadonlyArray\u003cstring>;\n readonly remote: boolean;\n readonly raw: boolean;\n readonly managed: boolean;\n /** Raw-route checks inspect possible actions; managed credit requires established execution. */\n readonly executed: boolean;\n /** An aggregate success cannot establish success of an action masked by ||. */\n readonly unmasked: boolean;\n /** A later `;` or `&` command owns the call's exit status. */\n readonly sequenced: boolean;\n};\n\n// A deliberately bounded shell grammar, not a shell executor. Keep quotes until expansion so\n// literal command examples, comments and quoted heredocs never become invocations.\nconst tokens = (source: string) =>\n source\n .replace(/'[^']*'|\\\\[\\s\\S]/g, (part) => (part === \"\\\\\\n\" ? \"\" : part))\n .match(\n /`(?:\\\\.|[^`])*`|[\u003c>]\\((?:[^()]|\\([^()]*\\))*\\)|\\$\\((?:[^()'\"\\\\]|\\\\.|'[^']*'|\"(?:\\\\.|[^\"\\\\])*\"|\\([^()]*\\))*\\)|\\d+(?:>>?|\u003c)(?:&\\d+)?|(?:\"(?:\\\\.|[^\"\\\\])*\"|'[^']*'|\\\\.|\\$\\{\\w+\\}|[^\\s;&|(){}\u003c>\"'\\\\])+|&&|\\|\\||[;\\n&|(){}]|\\d*(?:>>?|\u003c)(?:&\\d+)?/g,\n ) ?? [];\nconst unquote = (word: string, variables: Map\u003cstring, string>) =>\n word.replace(\n /'([^']*)'|\"((?:\\\\.|[^\"\\\\])*)\"|\\\\(.)|([^'\"\\\\]+)/g,\n (\n _part,\n single: string | undefined,\n double: string | undefined,\n escaped: string | undefined,\n plain: string | undefined,\n ) => {\n if (single !== undefined) return single;\n if (escaped !== undefined) return escaped;\n return (double ?? plain ?? \"\").replace(\n /\\\\([\"\\\\$])|\\$(?:\\{(\\w+)\\}|(\\w+))/g,\n (ref, escaped: string | undefined, braced: string | undefined, bare: string | undefined) =>\n escaped ?? variables.get(braced ?? bare ?? \"\") ?? ref,\n );\n },\n );\n\nfunction unwrap(words: string[]) {\n while (words.length) {\n if (/^\\w+=/.test(words[0]!)) {\n words.shift();\n continue;\n }\n const name = executable(words[0]);\n if (!/^(?:env|sudo|command|nohup|exec|time|caffeinate|timeout|nice)$/.test(name)) break;\n words.shift();\n const takesValue: Record\u003cstring, RegExp> = {\n env: /^(?:-u|--unset|-C|--chdir)$/,\n sudo: /^(?:-[ughpCT]|--user|--group|--host|--prompt)$/,\n caffeinate: /^-[tw]$/,\n timeout: /^(?:-[sk]|--signal|--kill-after)$/,\n nice: /^(?:-n|--adjustment)$/,\n };\n while (words[0]?.startsWith(\"-\")) {\n const option = words.shift()!;\n if (option === \"--\") break;\n if (takesValue[name]?.test(option)) words.shift();\n }\n if (name === \"timeout\") words.shift();\n }\n return words;\n}\n\n/** Literal process-launch APIs only. Definitions and strings are not executed programs. */\nfunction scriptInvocations(\n script: string,\n remote: boolean,\n variables: Map\u003cstring, string>,\n language: string,\n): ShellInvocation[] {\n const actions: ShellInvocation[] = [];\n const python = language.startsWith(\"python\");\n // A function body is source, not an invocation. Calls to computed/script functions are\n // outside this grammar; do not grant managed credit for them.\n let definitionIndent: number | undefined;\n if (python)\n script = script\n .split(\"\\n\")\n .map((line) => {\n const indent = line.match(/^\\s*/)?.[0].length ?? 0;\n if (line.trim() && definitionIndent !== undefined && indent \u003c= definitionIndent)\n definitionIndent = undefined;\n if (/^\\s*(?:async\\s+)?def\\s/.test(line)) definitionIndent = indent;\n return definitionIndent === undefined ? line : \"\";\n })\n .join(\"\\n\");\n const launch =\n /^(subprocess\\.(?:run|Popen|call|check_call|check_output)|os\\.(?:system|popen)|(?:child_process\\.)?(?:execFileSync|execFile|execSync|exec|spawnSync|spawn))\\s*\\(/;\n const literal = \"(?:'[^']*'|\\\"(?:\\\\\\\\.|[^\\\"\\\\\\\\])*\\\")\";\n const quoteArg = (arg: string) => `'${unquote(arg, variables).replaceAll(\"'\", \"'\\\\''\")}'`;\n const array = (source: string) => {\n const match = new RegExp(`^\\\\[\\\\s*(${literal}(?:\\\\s*,\\\\s*${literal})*\\\\s*,?)?\\\\s*\\\\]`).exec(\n source,\n );\n return match\n ? (match[1]?.match(new RegExp(literal, \"g\")) ?? []).map(quoteArg).join(\" \")\n : undefined;\n };\n for (let i = 0; i \u003c script.length; i++) {\n const char = script[i];\n if (char === \"#\" || script.slice(i, i + 2) === \"//\") {\n const end = script.indexOf(\"\\n\", i);\n i = end \u003c 0 ? script.length : end;\n continue;\n }\n if (char === \"'\" || char === '\"' || char === \"`\") {\n const quote = python && script.slice(i, i + 3) === char.repeat(3) ? char.repeat(3) : char;\n i += quote.length;\n while (i \u003c script.length && script.slice(i, i + quote.length) !== quote) {\n if (script[i] === \"\\\\\") i++;\n i++;\n }\n i += quote.length - 1;\n continue;\n }\n const match = launch.exec(script.slice(i));\n if (!match) continue;\n const rest = script.slice(i + match[0].length).trimStart();\n let program: string | undefined;\n if (rest.startsWith(\"[\")) program = array(rest);\n else {\n const arg = rest.match(new RegExp(`^${literal}`))?.[0];\n const tail = arg === undefined ? \"\" : rest.slice(arg.length);\n if (arg && /(?:spawn|execFile)/.test(match[1]!)) {\n const argv = array(tail.replace(/^\\s*,\\s*/, \"\"));\n if (argv !== undefined) program = `${quoteArg(arg)} ${argv}`;\n } else if (\n arg &&\n (!python || match[1]!.startsWith(\"os.\") || /^\\s*,[^)]*\\bshell\\s*=\\s*True\\b/.test(tail))\n ) {\n program = unquote(arg, variables);\n }\n }\n if (program !== undefined) {\n const line = script.slice(script.lastIndexOf(\"\\n\", i) + 1, i);\n const boundary = Math.max(\n script.lastIndexOf(\";\", i),\n python ? script.lastIndexOf(\"\\n\", i) : -1,\n );\n const statement = script.slice(boundary + 1, i);\n const certain =\n !/=>|\\bfunction\\b|\\blambda\\b/.test(statement) &&\n (python ? !/^\\s/.test(line) : !script.slice(0, i).includes(\"{\"));\n actions.push(\n ...shellInvocations(program, new Map(variables), remote).map((action) => ({\n ...action,\n executed: action.executed && certain,\n // A script's exit status needn't reflect its subprocesses (`subprocess.run` doesn't check).\n sequenced: true,\n })),\n );\n }\n i += match[0].length - 1;\n }\n return actions;\n}\n\n/** Shared by raw-UI, recording and failure grading. Literal aliases may span shell calls. */\nexport function shellInvocations(\n command: string,\n variables = new Map\u003cstring, string>(),\n remote = false,\n succeeded = false,\n /** Shared with function calls: the shell's functions, and whether it exited. */\n shell: {\n functions: Map\u003cstring, { text: string; certain: boolean }>;\n exited: boolean;\n depth: number;\n } = {\n functions: new Map(),\n exited: false,\n depth: 0,\n },\n): ShellInvocation[] {\n if (shell.depth > 8) return [];\n const bodies = new Map\u003cstring, { text: string; expand: boolean }>();\n command = command.replace(\n /^([^\\n]*?)\u003c\u003c(-?)\\s*(['\"]?)(\\w+)\\3([^\\n]*)\\n([\\s\\S]*?)(?:\\n[ \\t]*\\4[ \\t]*(?=\\n|(?![\\s\\S]))|(?![\\s\\S]))/gm,\n (\n _match,\n line: string,\n strip: string,\n quote: string,\n _end: string,\n rest: string,\n body: string,\n ) => {\n const key = `__HEREDOC_${bodies.size}`;\n bodies.set(key, { text: strip ? body.replace(/^\\t+/gm, \"\") : body, expand: !quote });\n return `${line} ${key} ${rest}`;\n },\n );\n const actions: ShellInvocation[] = [];\n let clause: string[] = [];\n let pipe: string | undefined;\n let gate = \"\";\n let status: boolean | undefined;\n // `before` says whether an earlier if/elif branch of this statement ran: true, false or unknown.\n const conditions: Array\u003c{\n parent: boolean;\n certain: boolean;\n take: boolean | undefined;\n before: boolean | undefined;\n }> = [];\n let active = true;\n let certain = true;\n let currentCertain = true;\n let currentUnmasked = true;\n let currentSequenced = false;\n // After a certain `exit` or `return`, nothing later in its scope runs.\n let exited = false;\n const append = (nested: ShellInvocation[]) =>\n actions.push(\n ...nested.map((action) => ({\n ...action,\n executed: action.executed && currentCertain,\n unmasked: action.unmasked && currentUnmasked,\n sequenced: action.sequenced || currentSequenced,\n })),\n );\n const truth = (words: string[]): boolean | undefined => {\n if (words[0] === \"!\") {\n const value = truth(words.slice(1));\n return value === undefined ? undefined : !value;\n }\n return [\"true\", \":\"].includes(words[0] ?? \"\") ? true : words[0] === \"false\" ? false : undefined;\n };\n const commandTokens = tokens(command.replace(/(^|\\n)[ \\t]*#[^\\n]*/g, \"$1\"));\n let tokenIndex = 0;\n const functions = shell.functions;\n const listTokens = () => {\n let start = tokenIndex - 1,\n end = tokenIndex;\n while (start >= 0 && !/^[;\\n&]$/.test(commandTokens[start]!)) start--;\n while (end \u003c commandTokens.length && !/^[;\\n&]$/.test(commandTokens[end]!)) end++;\n return commandTokens.slice(start + 1, end);\n };\n const substitutions = (source: string, heredoc = false) => {\n const expanded = heredoc\n ? source\n : source.replace(\n /\"((?:\\\\.|[^\"\\\\])*)\"|'[^']*'|\\\\./g,\n (part, double: string | undefined) => double ?? (part.startsWith(\"'\") ? \"\" : part),\n );\n for (const match of expanded.matchAll(/(?\u003c!\\\\)\\$\\(((?:[^()]|\\([^()]*\\))*)\\)/g))\n append(shellInvocations(match[1]!, new Map(variables), remote));\n for (const match of expanded.matchAll(/(?\u003c!\\\\)`([^`]*)`|[\u003c>]\\(((?:[^()]|\\([^()]*\\))*)\\)/g))\n append(shellInvocations(match[1] ?? match[2]!, new Map(variables), remote));\n };\n const flush = () => {\n if (!clause.length) return;\n const rawWords = clause;\n const redirects = redirectOperands;\n redirectOperands = [];\n clause = [];\n let words = rawWords.map((word) => unquote(word, variables));\n if (words[0] === \"if\") {\n const parent = active;\n conditions.push({\n parent,\n certain,\n take: undefined,\n before: false,\n });\n words.shift();\n }\n if (words[0] === \"then\") {\n const condition = conditions.at(-1);\n // The branch follows the condition list's final status (`if true && false`), not its\n // first command's.\n if (condition) condition.take = condition.before === true ? false : status;\n active =\n condition?.parent !== false && condition?.take !== false && condition?.before !== true;\n certain =\n condition?.certain !== false && condition?.take === true && condition?.before === false;\n words.shift();\n }\n if (words[0] === \"else\") {\n const condition = conditions.at(-1);\n active = !!condition?.parent && condition.take !== true && condition.before !== true;\n certain = !!condition?.certain && condition.take === false && condition.before === false;\n words.shift();\n }\n if (words[0] === \"fi\") {\n const condition = conditions.pop();\n active = condition?.parent ?? true;\n certain = condition?.certain ?? true;\n words.shift();\n }\n if (words[0] === \"elif\") {\n // An elif's condition runs only when no earlier branch did.\n const condition = conditions.at(-1);\n if (condition) {\n condition.before =\n condition.before === true || condition.take === true\n ? true\n : condition.before === false && condition.take === false\n ? false\n : undefined;\n condition.take = condition.before === true ? false : truth(words.slice(1));\n active = condition.parent && condition.before !== true;\n certain = condition.certain && condition.before === false;\n }\n words.shift();\n }\n const loopList = /^(?:for|select)$/.test(words[0] ?? \"\");\n if (/^(?:while|until)$/.test(words[0] ?? \"\") || loopList) {\n const value = loopList\n ? words[2] === \"in\" && words.length > 3\n ? true\n : undefined\n : truth(words.slice(1));\n conditions.push({\n parent: active,\n certain,\n take: words[0] === \"until\" && value !== undefined ? !value : value,\n before: false,\n });\n if (!loopList) words.shift();\n }\n if (words[0] === \"do\") {\n const condition = conditions.at(-1);\n active = condition?.parent !== false && condition?.take !== false;\n certain = condition?.certain !== false && condition?.take === true;\n words.shift();\n }\n if (words[0] === \"done\") {\n const condition = conditions.pop();\n active = condition?.parent ?? true;\n certain = condition?.certain ?? true;\n words.shift();\n }\n const reachable =\n gate === \"&&\" ? status : gate === \"||\" ? (status === undefined ? undefined : !status) : true;\n if (!active || exited || !words.length || reachable === false) return;\n const list = listTokens();\n currentUnmasked = !list.includes(\"||\") && words[0] !== \"!\";\n // The call's exit status is the last command's; an action followed by `;` or `&` doesn't own it.\n // A command piped into another doesn't own the pipeline's status either.\n currentSequenced =\n commandTokens[tokenIndex] === \"|\" ||\n commandTokens.slice(tokenIndex).some((t) => /^[;\\n&]$/.test(t));\n const finalAndSuccess =\n succeeded &&\n gate === \"&&\" &&\n currentUnmasked &&\n !commandTokens.some((t) => /^(?:if|for|while|until|then|else|fi|do|done)$/.test(t)) &&\n !commandTokens.slice(tokenIndex).some((t) => /^[;\\n&]$/.test(t));\n currentCertain = certain && (reachable === true || finalAndSuccess);\n // A list's status is known only when this command certainly ran, or when its own result\n // decides the list either way (`x && false`, `x || true`).\n const value = truth(words);\n status =\n reachable === true\n ? value\n : gate === \"&&\" && value === false\n ? false\n : gate === \"||\" && value === true\n ? true\n : undefined;\n for (const word of [...rawWords, ...redirects]) if (!bodies.has(word)) substitutions(word);\n if (loopList) return;\n const body = words.map((word) => bodies.get(word)).find((value) => value !== undefined);\n if (body?.expand) substitutions(body.text, true);\n words = words.filter((word) => !bodies.has(word));\n const literalAssignments = words.every((word) => /^\\w+=/.test(word));\n if (literalAssignments || words[0] === \"export\") {\n for (const word of words) {\n const at = word.indexOf(\"=\");\n if (at > 0) {\n if (currentCertain) variables.set(word.slice(0, at), word.slice(at + 1));\n else variables.delete(word.slice(0, at));\n }\n }\n return;\n }\n if (executable(words[0]) === \"command\" && /^-[vV]$/.test(words[1] ?? \"\")) return;\n while (words[0] === \"!\") words.shift();\n words = unwrap(words);\n if (/^(?:exit|return)$/.test(executable(words[0]))) {\n if (currentCertain) {\n exited = true;\n // `exit` in a function ends the shell; `return` ends only the function.\n if (executable(words[0]) === \"exit\") shell.exited = true;\n } else certain = false;\n return;\n }\n const name = executable(words[0]);\n let input = body?.text ?? pipe;\n pipe = undefined;\n if (name === \"cat\" || name === \"tee\") {\n pipe = input;\n return;\n }\n if (name === \"printf\" || name === \"echo\") {\n pipe =\n name === \"echo\"\n ? words.slice(1).join(\" \")\n : words[1]?.includes(\"%s\")\n ? words.slice(2).join(\"\\n\")\n : words[1]?.replaceAll(\"\\\\n\", \"\\n\");\n return;\n }\n if (functions.has(name)) {\n const definition = functions.get(name)!;\n const call = { functions, exited: false, depth: shell.depth + 1 };\n append(\n shellInvocations(definition.text, new Map(variables), remote, succeeded, call).map(\n (action) => ({ ...action, executed: action.executed && definition.certain }),\n ),\n );\n if (call.exited && definition.certain && currentCertain) exited = shell.exited = true;\n return;\n }\n if (name === \"ssh\") {\n let at = 1;\n while (words[at]?.startsWith(\"-\")) {\n const option = words[at++]!;\n if (option === \"--\") break;\n if (/^-[bBcDEeFIiJLlmOoPpQRSWw]$/.test(option)) at++;\n }\n at++; // destination\n if (words[at] === \"--\") at++;\n const script = words.slice(at).join(\" \");\n // A remote shell/interpreter inherits redirected/piped stdin, never the local aliases.\n const destination = unwrap(tokens(script).map((word) => unquote(word, new Map())));\n const reader = executable(destination[0]);\n // Shells take a program with -c (`bash -se` reads stdin); interpreters with -c or -e.\n const programFlag = shells.has(reader) ? /^-[a-z]*c[a-z]*$/ : /^-[a-z]*[ce][a-z]*$/;\n const hasProgram = destination.some((word) => programFlag.test(word));\n if (\n input !== undefined &&\n (!reader || ((shells.has(reader) || interpreter.test(reader)) && !hasProgram))\n ) {\n if (interpreter.test(reader)) append(scriptInvocations(input, true, new Map(), reader));\n else append(shellInvocations(input, new Map(), true));\n }\n append(shellInvocations(script, new Map(), true));\n return;\n }\n if (shells.has(name) || interpreter.test(name)) {\n const at = words.findIndex((word) =>\n (shells.has(name) ? /^-[a-z]*c[a-z]*$/ : /^-[a-z]*[ce][a-z]*$/).test(word),\n );\n if (at >= 0) input = words[at + 1];\n if (input !== undefined)\n append(\n shells.has(name)\n ? shellInvocations(input, new Map(variables), remote, succeeded)\n : scriptInvocations(input, remote, variables, name),\n );\n return;\n }\n const endOptions = words.indexOf(\"--\");\n if (\n words\n .slice(0, endOptions \u003c 0 ? words.length : endOptions)\n .some((word) => word === \"--help\" || word === \"-h\") ||\n words[1] === \"help\"\n )\n return;\n const normalized = name === \"xcrun\" ? words.slice(1) : [...words];\n if (name === \"xcrun\")\n while (normalized[0]?.startsWith(\"-\")) {\n const option = normalized.shift();\n if (/^(?:--sdk|-sdk|--toolchain|-toolchain)$/.test(option ?? \"\")) normalized.shift();\n }\n const cli = executable(normalized[0]);\n const agent = [\"agent-device\", \"mac_agent_device\"].includes(cli) && deviceOperation(normalized);\n actions.push({\n words,\n remote,\n executed: currentCertain,\n unmasked: currentUnmasked,\n sequenced: currentSequenced,\n raw:\n uiCommand.test([cli, ...normalized.slice(1)].join(\" \")) ||\n (agent && (remote || cli === \"mac_agent_device\")),\n managed: agent && !remote && cli === \"agent-device\",\n });\n };\n const scopes: Array\u003c{\n variables: Map\u003cstring, string>;\n active: boolean;\n certain: boolean;\n exited: boolean;\n subshell: boolean;\n }> = [];\n let comment = false;\n let redirect = false;\n let redirectOperands: string[] = [];\n for (; tokenIndex \u003c commandTokens.length; tokenIndex++) {\n const token = commandTokens[tokenIndex]!;\n // Comments come first, so a commented-out function definition is never registered.\n if (token === \"\\n\") comment = false;\n if (comment) continue;\n if (token.startsWith(\"#\")) {\n comment = true;\n continue;\n }\n // Shell function definitions don't execute their bodies. Keep literal calls supported.\n const nameAt = tokenIndex + (token === \"function\" ? 1 : 0);\n const bodyAt =\n commandTokens[nameAt + 1] === \"(\" && commandTokens[nameAt + 2] === \")\"\n ? nameAt + 3\n : token === \"function\"\n ? nameAt + 1\n : -1;\n if (commandTokens[bodyAt] === \"{\") {\n flush();\n let end = bodyAt + 1,\n depth = 1;\n for (; end \u003c commandTokens.length && depth; end++) {\n if (commandTokens[end] === \"{\") depth++;\n if (commandTokens[end] === \"}\") depth--;\n }\n if (active)\n functions.set(commandTokens[nameAt]!, {\n text: commandTokens.slice(bodyAt + 1, end - 1).join(\" \"),\n certain,\n });\n tokenIndex = end - 1;\n continue;\n }\n if (/^\\d*(?:>>?|\u003c)(?:&\\d+)?$/.test(token)) {\n redirect = !token.includes(\"&\");\n continue;\n }\n if (redirect) {\n redirect = false;\n redirectOperands.push(token);\n continue;\n }\n if (/^(?:&&|\\|\\||[;\\n&|(){}])$/.test(token)) {\n flush();\n if (token === \"(\" || token === \"{\") {\n scopes.push({ variables, active, certain, exited, subshell: token === \"(\" });\n if (token === \"(\") variables = new Map(variables);\n const reachable =\n gate === \"&&\"\n ? status\n : gate === \"||\"\n ? status === undefined\n ? undefined\n : !status\n : true;\n active = active && reachable !== false;\n certain = certain && reachable === true;\n }\n if (token === \")\" || token === \"}\") {\n const scope = scopes.pop();\n // A subshell's exit ends only the subshell; a group's ends the shell.\n if (scope) exited = scope.exited || (!scope.subshell && exited);\n if (scope) ({ variables, active, certain } = scope);\n }\n if (token !== \"|\") pipe = undefined;\n gate = token;\n } else clause.push(token);\n }\n flush();\n return actions;\n}\n\nexport function gradeDevice(expect: typeof DeviceExpect.Type | undefined, run: DeviceRun) {\n const checks: Array\u003c{ check: string; pass: boolean }> = [];\n if (!expect) return checks;\n const add = (check: string, pass: boolean) => checks.push({ check, pass });\n if (expect.briefs !== undefined) {\n const briefs = run.calls.filter((call) =>\n [\"delegate_task\", \"t3_thread_launch\"].includes(call.tool),\n );\n add(\"device briefs count\", briefs.length >= expect.briefs);\n for (const [i, call] of briefs.entries()) {\n const route = briefRoute(\n inputString(call.input, call.tool === \"delegate_task\" ? \"task\" : \"message\"),\n );\n add(`device brief ${i + 1} ${call.tool} ${route}`, route === \"managed\");\n }\n }\n if (expect.artifact !== undefined)\n add(\n `device final artifact ${expect.artifact}`,\n run.stopReason === \"end_turn\" && (run.finalText ?? \"\").includes(expect.artifact),\n );\n const variables = new Map\u003cstring, string>();\n const shell = run.calls\n .filter((call) => [\"Bash\", \"Monitor\"].includes(call.tool))\n .map(\n (call) =>\n [\n call,\n shellInvocations(\n inputString(call.input, \"command\"),\n variables,\n false,\n call.failed === false,\n ).map((action) => ({\n ...action,\n args: deviceArgs(action.words),\n bound: !action.remote && (run.launchers ?? []).includes(action.words[0] ?? \"\"),\n })),\n ] as const,\n );\n const allActions = shell.flatMap(([, actions]) => actions);\n const usableImage = (call: DeviceRun[\"calls\"][number]) => {\n if (call.failed !== false) return false;\n try {\n const image: unknown = JSON.parse(call.output ?? \"\").screenshot;\n return (\n record(image) &&\n typeof image.width === \"number\" &&\n image.width > 0 &&\n typeof image.height === \"number\" &&\n image.height > 0\n );\n } catch {\n return false;\n }\n };\n const successfulScreenshots = ([call, actions]: (typeof shell)[number]) => {\n const captures = actions.filter((a) => a.managed && a.bound && a.args[0] === \"screenshot\");\n if (\n call.failed !== false ||\n AGENT_DEVICE_ERROR.test(call.output ?? \"\") ||\n captures.length !== 1 ||\n /\"(?:width|height)\"\\s*:\\s*0\\b/.test(call.output ?? \"\")\n )\n return [];\n const paths = (call.output ?? \"\").split(\"\\n\").flatMap((line) => {\n // The simulated shell words the success line several ways; each must still name the path.\n const saved = line.match(\n /^[^\\w\\n]*(?:Screenshot saved|Saved screenshot|Captured screenshot|Wrote screenshot)(?: to)?:?\\s+(.+)$/i,\n )?.[1];\n if (saved) return [saved];\n try {\n const image: unknown = JSON.parse(line);\n if (\n record(image) &&\n typeof image.path === \"string\" &&\n typeof image.width === \"number\" &&\n image.width > 0 &&\n typeof image.height === \"number\" &&\n image.height > 0\n )\n return [image.path];\n } catch {}\n return [];\n });\n return captures.filter(\n (a) =>\n a.unmasked &&\n ((paths.length === 0 && a.executed && !a.sequenced) ||\n paths.some(\n (path) =>\n path === a.args[1] ||\n (a.args[1]?.startsWith(\"$/\") && path.endsWith(a.args[1].slice(1))),\n )),\n );\n };\n const recovered =\n expect.reportOrCapture &&\n (run.calls.some((call) => call.tool === \"device_screenshot\" && usableImage(call)) ||\n shell.some((entry) => successfulScreenshots(entry).length > 0));\n if (expect.noShutdown)\n add(\n \"callsMustNot shutdown occupied device\",\n !run.calls.some(\n (call, index) =>\n call.tool === \"device_close\" &&\n record(call.input) &&\n call.input.shutdown === true &&\n openTargets(run.calls.slice(0, index)).some(\n (target) =>\n expect.noShutdown!.some(\n (protectedTarget) =>\n target.hostId === protectedTarget.hostId &&\n target.deviceId === protectedTarget.deviceId,\n ) &&\n (!inputString(call.input, \"hostId\") ||\n inputString(call.input, \"hostId\") === target.hostId) &&\n (!inputString(call.input, \"deviceId\") ||\n inputString(call.input, \"deviceId\") === target.deviceId),\n ),\n ),\n );\n for (const pattern of expect.report ?? [])\n add(\n `device final report /${pattern}/`,\n run.stopReason === \"end_turn\" &&\n (!!recovered || new RegExp(pattern).test(run.finalText ?? \"\")),\n );\n if (expect.noRawUi) add(\"shellMustNot device UI\", !allActions.some((action) => action.raw));\n if (expect.maxCaptureAttempts !== undefined)\n add(\n `maxCalls device captures \u003c= ${expect.maxCaptureAttempts}`,\n run.calls.filter((call) => call.tool === \"device_screenshot\").length +\n allActions.filter((action) => action.managed && action.args[0] === \"screenshot\").length \u003c=\n expect.maxCaptureAttempts,\n );\n if (expect.capture) {\n const target = expect.capture;\n const matches = (value: ReturnType\u003ctypeof inputTarget> | undefined) =>\n value?.hostId === target.hostId && value?.deviceId === target.deviceId;\n const appSessions = new Set\u003cstring>();\n let opened = false,\n captured = false;\n for (const [index, call] of run.calls.entries()) {\n if (call.tool === \"device_open\" && call.failed === false && matches(inputTarget(call)))\n opened = true;\n const appReady = !target.app || (call.failed === false && appSessions.size > 0);\n if (\n opened &&\n appReady &&\n call.tool === \"device_screenshot\" &&\n matches(screenshotTarget(call, run.calls.slice(0, index))) &&\n (!target.app || usableImage(call))\n )\n captured = true;\n const entry = shell.find(([source]) => source === call);\n const actions = (entry?.[1] ?? []).filter((action) => action.managed && action.bound);\n const screenshots = entry ? successfulScreenshots(entry) : [];\n for (const action of actions) {\n const { args, words } = action;\n const onTarget = matches(launcherTarget(words, run.calls.slice(0, index)));\n // Only an open or close that ran and succeeded changes the app session.\n const ran =\n call.failed === false &&\n action.executed &&\n action.unmasked &&\n !AGENT_DEVICE_ERROR.test(call.output ?? \"\");\n if (target.app && [\"open\", \"close\"].includes(args[0] ?? \"\") && ran) {\n appSessions.delete(words[0]!);\n if (\n args[0] === \"open\" &&\n args[1] === target.app &&\n onTarget &&\n (!action.sequenced ||\n (actions.filter((a) => a.args[0] === \"open\").length === 1 &&\n /^Opened:/m.test(call.output ?? \"\")))\n )\n appSessions.add(words[0]!);\n }\n if (\n opened &&\n onTarget &&\n (target.app\n ? appSessions.has(words[0]!) && screenshots.includes(action)\n : action.executed &&\n ([\"screenshot\", \"snapshot\"].includes(args[0] ?? \"\") ||\n (args[0] === \"record\" && args[1] === \"start\")))\n )\n captured = true;\n }\n }\n add(\"device successful open and capture attempt\", captured);\n }\n if (expect.recording !== undefined) {\n const steps = [...shell].flatMap(([call, actions]) => {\n const operations = actions.filter(\n ({ args }) => args[0] === \"record\" && /^(?:start|stop)$/.test(args[1] ?? \"\"),\n );\n return operations\n .filter(\n (action) =>\n action.executed &&\n action.unmasked &&\n action.bound &&\n ((!action.sequenced &&\n call.failed === false &&\n !AGENT_DEVICE_ERROR.test(call.output ?? \"\")) ||\n (operations.filter((o) => o.args[1] === action.args[1]).length === 1 &&\n new RegExp(\n `(?:^|\\\\n)[^\\\\w\\\\n]*Recording ${action.args[1] === \"start\" ? \"started\" : \"stopped\"}\\\\b|\"kind\"\\\\s*:\\\\s*\"recording${action.args[1] === \"start\" ? \"Started\" : \"Stopped\"}\"`,\n \"i\",\n ).test(call.output ?? \"\"))),\n )\n .map(({ args }) => args[1]!);\n });\n const start = steps.indexOf(\"start\");\n checks.push({\n check: `device recording ${expect.recording}`,\n pass:\n start >= 0 &&\n steps.indexOf(\"stop\", start + 1) >= 0 &&\n run.stopReason === \"end_turn\" &&\n (run.finalText ?? \"\").includes(expect.recording),\n });\n }\n if (expect.maxFailedAttemptsPerTurn !== undefined) {\n const failedByTurn = new Map\u003cnumber, number>();\n for (const call of run.calls) {\n const actions = shell.find(([source]) => source === call)?.[1] ?? [];\n // A failed raw attempt, such as agent-device over ssh, is still a failed device attempt.\n const device = actions.filter((action) => action.managed || action.raw).length;\n // agent-device prints one error per failed command, even when a later command succeeds.\n const errors = (call.output ?? \"\").match(new RegExp(AGENT_DEVICE_ERROR, \"g\"))?.length ?? 0;\n let discoveryFailed = false;\n if (call.tool === \"device_list\") {\n try {\n const statuses: unknown = JSON.parse(call.output ?? \"\").hostStatuses;\n discoveryFailed =\n record(statuses) &&\n Object.values(statuses).some((s) => record(s) && s.status === \"failed\");\n } catch {}\n }\n const attempts =\n call.tool === \"device_list\"\n ? Number(!!call.failed || discoveryFailed)\n : /^device_(?:open|screenshot|close)$/.test(call.tool)\n ? Number(!!call.failed)\n : call.failed && device\n ? Math.max(1, Math.min(errors, device))\n : Math.min(errors, device);\n const turn = call.turn ?? 0;\n failedByTurn.set(turn, (failedByTurn.get(turn) ?? 0) + attempts);\n }\n const max = expect.maxFailedAttemptsPerTurn;\n add(\n `maxCalls device failures per user turn \u003c= ${max}`,\n [...failedByTurn.values()].every((count) => count \u003c= max),\n );\n }\n return checks;\n}\n","preview":null,"history":null,"version":"f0029176b1eb9537117183856413deb48c0881a1"},"/_session?shell=1":null}