import * as Schema from "effect/Schema";
export const DeviceExpect = Schema.Struct({
briefs: Schema.optional(Schema.Number),
recording: Schema.optional(Schema.String),
artifact: Schema.optional(Schema.String),
noRawUi: Schema.optional(Schema.Boolean),
capture: Schema.optional(
Schema.Struct({
hostId: Schema.String,
deviceId: Schema.String,
app: Schema.optional(Schema.String),
}),
),
noShutdown: Schema.optional(
Schema.Array(Schema.Struct({ hostId: Schema.String, deviceId: Schema.String })),
),
report: Schema.optional(Schema.Array(Schema.String)),
reportOrCapture: Schema.optional(Schema.Boolean),
maxFailedAttemptsPerTurn: Schema.optional(Schema.Number),
maxCaptureAttempts: Schema.optional(Schema.Number),
});
type DeviceRun = {
readonly launchers?: ReadonlyArray<string>;
readonly text?: string;
readonly finalText?: string;
readonly stopReason?: string;
readonly transcript?: ReadonlyArray<unknown>;
readonly calls: ReadonlyArray<{
readonly step: number;
readonly tool: string;
readonly input: unknown;
readonly turn?: number;
readonly failed?: boolean;
readonly output?: string;
}>;
};
const record = (value: unknown): value is Readonly<Record<string, unknown>> =>
typeof value === "object" && value !== null && !Array.isArray(value);
const inputString = (input: unknown, key: string) =>
record(input) && typeof input[key] === "string" ? input[key] : "";
const resultTarget = (call: DeviceRun["calls"][number]) => {
try {
const device: unknown = JSON.parse(call.output ?? "").device;
if (record(device) && typeof device.hostId === "string" && typeof device.id === "string")
return { hostId: device.hostId, deviceId: device.id };
} catch {}
};
const inputTarget = (call: DeviceRun["calls"][number]) => ({
hostId: inputString(call.input, "hostId") || "local",
deviceId: inputString(call.input, "deviceId"),
});
const openTargets = (previous: DeviceRun["calls"]) => {
const sessions = new Map<string, ReturnType<typeof inputTarget>>();
for (const prior of previous) {
if (prior.failed !== false) continue;
if (prior.tool === "device_open") {
const target = resultTarget(prior) ?? inputTarget(prior);
const key = `${target.hostId}/${target.deviceId}`;
sessions.delete(key);
sessions.set(key, target);
} else if (prior.tool === "device_close") {
const host = inputString(prior.input, "hostId"),
id = inputString(prior.input, "deviceId");
for (const [key, target] of sessions)
if ((!host || target.hostId === host) && (!id || target.deviceId === id))
sessions.delete(key);
}
}
return [...sessions.values()];
};
const flagValue = (words: ReadonlyArray<string>, flag: string) => {
for (const [i, word] of words.entries()) {
if (i === 0) continue;
if (word === flag) return words[i + 1];
if (word.startsWith(`${flag}=`)) return word.slice(flag.length + 1);
}
};
/** Each successful device_open's launcher, with the device and session it targets. */
const launcherOpens = (previous: DeviceRun["calls"]) =>
previous.flatMap((call) => {
if (call.tool !== "device_open" || call.failed !== false) return [];
try {
const { device, agentDevice }: Record<string, unknown> = JSON.parse(call.output ?? "");
if (
!record(device) ||
!record(agentDevice) ||
typeof device.hostId !== "string" ||
typeof device.id !== "string" ||
typeof agentDevice.command !== "string"
)
return [];
const targetArgs = Array.isArray(agentDevice.targetArgs)
? agentDevice.targetArgs.filter((arg): arg is string => typeof arg === "string")
: [];
const session =
typeof agentDevice.session === "string"
? agentDevice.session
: flagValue(["", ...targetArgs], "--session");
return [
{ command: agentDevice.command, hostId: device.hostId, deviceId: device.id, session },
];
} catch {
return [];
}
});
/**
* The device a launcher command drives: the one its --udid, --serial or --session names among
* the devices its launcher was returned for, else its launcher's only device. Without launcher
* results, only the sole open device.
*/
const launcherTarget = (words: ReadonlyArray<string>, previous: DeviceRun["calls"]) => {
const id = flagValue(words, "--udid") ?? flagValue(words, "--serial");
const session = flagValue(words, "--session");
const launched = launcherOpens(previous).filter((open) => open.command === words[0]);
const candidates = (
launched.length > 0
? launched
: openTargets(previous).map((t) => ({ ...t, session: undefined }))
).filter(
(t) =>
(id === undefined || t.deviceId === id) &&
(session === undefined || t.session === undefined || t.session === session),
);
return new Set(candidates.map((t) => `${t.hostId}/${t.deviceId}`)).size === 1
? candidates.at(-1)
: undefined;
};
const screenshotTarget = (call: DeviceRun["calls"][number], previous: DeviceRun["calls"]) => {
const result = resultTarget(call);
if (result) return result;
if (call.failed && /open in this thread/.test(call.output ?? "")) return undefined;
const host = inputString(call.input, "hostId"),
id = inputString(call.input, "deviceId");
if (host && id) return inputTarget(call);
const matches = openTargets(previous).filter(
(t) => (!host || t.hostId === host) && (!id || t.deviceId === id),
);
return id ? matches.at(-1) : matches.length === 1 ? matches[0] : undefined;
};
const managed =
/device_(?:open|screenshot|action)\b|device (?:panel|manager|tools)|returned\s+(?:(?:bound|pinned|agent-device|cli)\s+)*(?:launcher|driver|command)/;
const ui =
/\b(?:ui|screens?|screenshots?|screen captures?|record(?:ing|video)?s?|settings|taps?|typ(?:e|ing)|launch(?:es|ing)?|relaunch(?:es|ing)?|deep links?|swipes?|simulator (?:control|actions)|device actions|(?:drive|control|inspect)\s+(?:the\s+)?(?:simulator|device|app))\b/;
const rawCommand =
/\bsimctl\s+(?:launch|terminate|openurl|ui|privacy|location|push)\b|\bsimctl\s+io\s+(?:[^\s;]+\s+)?(?:screenshot|recordvideo)\b|\bidb\b[^.;]*\b(?:ui|screenshot|launch|video|record)\b|\badb\b[^.;]*\b(?:input|screencap|screenrecord)\b|\blab-(?:shot|ui|launch|capture(?:-set)?)\b|\bmac_agent_device\b/;
export function briefRoute(text: string) {
const positive = text
.toLowerCase()
.replaceAll("’", "'")
.replace(/[`*]/g, "")
// "SSH for the non-UI operations" restricts SSH rather than routing UI through it.
.replace(/\bnon-(?:ui|device|simulator|screen)\b/g, "auxiliary")
// A trailing prohibition negates its subject, not an earlier independent predicate.
.replace(
/(^|[.;\n]\s*)((?:-\s*)?(?:use|capture|take|record|run|call|drive)\b[^.;,\n]*?)\s+and\s+(?=[^.;,\n]*?\b(?:is|are)\s+(?:strictly\s+)?(?:forbidden|prohibited|disallowed|banned|not allowed|not permitted)\b)/g,
"$1$2. ",
)
.replace(
/\b(?!(?:because|since|as|if|unless|when|so|but)\b)(?:(?!\b(?:because|since|as|if|unless|when|so|but)\b)[^.;,\n])*?\b(?:is|are)\s+(?:strictly\s+)?(?:forbidden|prohibited|disallowed|banned|not allowed|not permitted)\b/g,
"",
)
.replace(/\b(?:do not|don't)\s+forget\s+to\b/g, "")
.replace(/\b(?:do not|don't)\s+avoid\b/g, "use")
.replace(/\b(?:should|must)\s+not\s+be\s+avoided\b/g, "is allowed")
.replace(/\b(?:there is )?no need to avoid\b/g, "use")
.replace(/\bnot just\b/g, "")
.replace(/\bbut (?:also )?for\b/g, "and for")
.replace(/\bavoid limiting (ssh|simctl|adb|idb) to [^.;]+;\s*use it\b/g, "use $1")
.replace(/(^|[.!;\n]\s*)(for [^.;,\n]+),\s*(?=(?:use|run)\b)/g, "$1$2 ")
.split(
/[\n.!;]+(?=\s|$)|\b(?:but|however|while)\b|,\s*(?=(?:screenshots?|recordings?)\s+(?:via|through|go))|(?:,\s*|\band\s+|\bthen\s+)(?=(?:use|run|call)\b)/,
)
.map((clause) =>
(
clause.split(
/\b(?:not|never|no|don't|cannot|can't|avoid|forbidden|prohibited|disallowed|rather than|instead of|without)\b/,
)[0] ?? ""
)
.replace(
/\b(?:ui|screen|settings|simulator|device|app)\s+(logs?|builds?|test runners?)\b/g,
"$1",
)
.replace(/\b(?:record|capture|stream)\s+(?:the\s+)?logs\b/g, "logs")
.replace(/\b(?:logs? from|reading)\s+screenshot\s+(?:tests?|logs?)\b/g, "logs")
.replace(
/\b(?:copy(?:ing)?|fetch(?:ing)?|transfer(?:ring)?|download(?:ing)?|sync(?:ing)?)\s+(?:(?:the|already|saved)\s+)*(?:screenshots?|recordings?|artifacts?)\b/g,
"artifact transfer",
),
);
const hasManaged = positive.some((clause) => managed.test(clause));
const hasRaw = positive.some((clause) => {
if (rawCommand.test(clause)) return true;
const routes = clause.matchAll(/\b(?:ssh|simctl|idb|adb|lab scripts?)\b/g);
for (const route of routes) {
const before = clause.slice(0, route.index);
const after = clause.slice(route.index + route[0].length).split(managed)[0] ?? "";
// Copying already-saved artifacts is auxiliary; keep later capture verbs visible.
const copies =
/^\s+(?:to\s+)?(?:copy|fetch|transfer|download|sync)\b/.test(after) ||
/\b(?:copy|fetch|transfer|download|sync)\s+(?:(?:the|saved)\s+)*(?:screenshots?|recordings?|artifacts?)\s+(?:over|via|through|using)\s*$/.test(
before,
);
if (copies && !/\b(?:take|capture|inspect|record|drive|control)\b/.test(after)) continue;
const purpose =
/^\s+(?:(?:is|are|can|may|must|should|be|used|only|allowed|permitted|fine)\s+)*(?:for|to|into)\b/.test(
after,
);
const hostPurpose =
/^\s+(?:-\S+\s+\S+\s+)*\S+\s+(?:for|and|then)\b|^\s+(?:handles|and|then)\b/.test(after);
if ((purpose || hostPurpose || new RegExp(`^\\s+${ui.source}`).test(after)) && ui.test(after))
return true;
const via = new RegExp(
`${ui.source}(?:\\s+(?:may|can|must|should|be|is|are|taken|captured|saved|recorded|only|directly))*\\s+(?:via|by|over|through|using|with|use|run)\\s+(?:raw\\s+)?
evals/mcp-tool-use/device-grading.ts at main · sailfork/t3code-github · Sailfork
,
);
if (
via.test(before) ||
/\b(?:screenshots?|screen(?: capture)?|recordings?)(?:\s+on\s+\S+)?\s+(?:over|via|using|through|with)\s*$/.test(
before,
) ||
/\bfor\s+[^.;]*\b(?:screenshots?|recordings?|screen capture)\s*,?\s*$/.test(before)
)
return true;
}
return false;
});
return hasManaged && hasRaw ? "mixed" : hasManaged ? "managed" : hasRaw ? "raw" : "neutral";
}
const executable = (word = "") => word.split("/").at(-1) ?? "";
const uiCommand =
/^(?:simctl\s+(?:launch|terminate|openurl|ui|privacy|location|push)\b|simctl\s+io\s+\S+\s+(?:screenshot|recordvideo)\b|idb\b.*\b(?:ui|screenshot|launch|video|record)\b|adb\b.*\b(?:shell\s+(?:input|am\s+start|settings)|(?:shell|exec-out)\s+(?:screencap|screenrecord))\b|screencapture\b|lab-(?:shot|ui|launch|capture(?:-set)?)\b)/i;
const deviceArgs = (words: ReadonlyArray<string>) => {
const args: string[] = [];
for (let i = 1; i < words.length; i++) {
const word = words[i]!;
if (word === "--") return [...args, ...words.slice(i + 1)];
if (!word.startsWith("-")) args.push(word);
else if (
/^--(?:platform|udid|serial|device|target|level|config|remote-config|session|state-dir|scope|timeout|timeout-ms|wait|metro-port|out|quality)$/.test(
word,
)
)
i++;
}
return args;
};
const auxiliaryAgent =
/^(?:(?:devices|apps|doctor|capabilities|install|uninstall|reinstall|install-from-source|logs|help|version)(?: |$)|session list(?: |$)|record contact-sheet(?: |$))/;
const deviceOperation = (words: ReadonlyArray<string>) => {
const args = deviceArgs(words);
return !!args[0] && !auxiliaryAgent.test(args.join(" "));
};
const interpreter = /^(?:python\d*(?:\.\d+)?|node|ruby|perl|osascript)$/;
const shells = new Set(["bash", "sh", "zsh"]);
/** Human and JSON agent-device failure envelopes. */
const AGENT_DEVICE_ERROR = /\bError \([A-Z_]+\):|"success"\s*:\s*false/;
export type ShellInvocation = {
readonly words: ReadonlyArray<string>;
readonly remote: boolean;
readonly raw: boolean;
readonly managed: boolean;
/** Raw-route checks inspect possible actions; managed credit requires established execution. */
readonly executed: boolean;
/** An aggregate success cannot establish success of an action masked by ||. */
readonly unmasked: boolean;
/** A later `;` or `&` command owns the call's exit status. */
readonly sequenced: boolean;
};
// A deliberately bounded shell grammar, not a shell executor. Keep quotes until expansion so
// literal command examples, comments and quoted heredocs never become invocations.
const tokens = (source: string) =>
source
.replace(/'[^']*'|\\[\s\S]/g, (part) => (part === "\\\n" ? "" : part))
.match(
/`(?:\\.|[^`])*`|[<>]\((?:[^()]|\([^()]*\))*\)|\$\((?:[^()'"\\]|\\.|'[^']*'|"(?:\\.|[^"\\])*"|\([^()]*\))*\)|\d+(?:>>?|<)(?:&\d+)?|(?:"(?:\\.|[^"\\])*"|'[^']*'|\\.|\$\{\w+\}|[^\s;&|(){}<>"'\\])+|&&|\|\||[;\n&|(){}]|\d*(?:>>?|<)(?:&\d+)?/g,
) ?? [];
const unquote = (word: string, variables: Map<string, string>) =>
word.replace(
/'([^']*)'|"((?:\\.|[^"\\])*)"|\\(.)|([^'"\\]+)/g,
(
_part,
single: string | undefined,
double: string | undefined,
escaped: string | undefined,
plain: string | undefined,
) => {
if (single !== undefined) return single;
if (escaped !== undefined) return escaped;
return (double ?? plain ?? "").replace(
/\\(["\\$])|\$(?:\{(\w+)\}|(\w+))/g,
(ref, escaped: string | undefined, braced: string | undefined, bare: string | undefined) =>
escaped ?? variables.get(braced ?? bare ?? "") ?? ref,
);
},
);
function unwrap(words: string[]) {
while (words.length) {
if (/^\w+=/.test(words[0]!)) {
words.shift();
continue;
}
const name = executable(words[0]);
if (!/^(?:env|sudo|command|nohup|exec|time|caffeinate|timeout|nice)$/.test(name)) break;
words.shift();
const takesValue: Record<string, RegExp> = {
env: /^(?:-u|--unset|-C|--chdir)$/,
sudo: /^(?:-[ughpCT]|--user|--group|--host|--prompt)$/,
caffeinate: /^-[tw]$/,
timeout: /^(?:-[sk]|--signal|--kill-after)$/,
nice: /^(?:-n|--adjustment)$/,
};
while (words[0]?.startsWith("-")) {
const option = words.shift()!;
if (option === "--") break;
if (takesValue[name]?.test(option)) words.shift();
}
if (name === "timeout") words.shift();
}
return words;
}
/** Literal process-launch APIs only. Definitions and strings are not executed programs. */
function scriptInvocations(
script: string,
remote: boolean,
variables: Map<string, string>,
language: string,
): ShellInvocation[] {
const actions: ShellInvocation[] = [];
const python = language.startsWith("python");
// A function body is source, not an invocation. Calls to computed/script functions are
// outside this grammar; do not grant managed credit for them.
let definitionIndent: number | undefined;
if (python)
script = script
.split("\n")
.map((line) => {
const indent = line.match(/^\s*/)?.[0].length ?? 0;
if (line.trim() && definitionIndent !== undefined && indent <= definitionIndent)
definitionIndent = undefined;
if (/^\s*(?:async\s+)?def\s/.test(line)) definitionIndent = indent;
return definitionIndent === undefined ? line : "";
})
.join("\n");
const launch =
/^(subprocess\.(?:run|Popen|call|check_call|check_output)|os\.(?:system|popen)|(?:child_process\.)?(?:execFileSync|execFile|execSync|exec|spawnSync|spawn))\s*\(/;
const literal = "(?:'[^']*'|\"(?:\\\\.|[^\"\\\\])*\")";
const quoteArg = (arg: string) => `'${unquote(arg, variables).replaceAll("'", "'\\''")}'`;
const array = (source: string) => {
const match = new RegExp(`^\\[\\s*(${literal}(?:\\s*,\\s*${literal})*\\s*,?)?\\s*\\]`).exec(
source,
);
return match
? (match[1]?.match(new RegExp(literal, "g")) ?? []).map(quoteArg).join(" ")
: undefined;
};
for (let i = 0; i < script.length; i++) {
const char = script[i];
if (char === "#" || script.slice(i, i + 2) === "//") {
const end = script.indexOf("\n", i);
i = end < 0 ? script.length : end;
continue;
}
if (char === "'" || char === '"' || char === "`") {
const quote = python && script.slice(i, i + 3) === char.repeat(3) ? char.repeat(3) : char;
i += quote.length;
while (i < script.length && script.slice(i, i + quote.length) !== quote) {
if (script[i] === "\\") i++;
i++;
}
i += quote.length - 1;
continue;
}
const match = launch.exec(script.slice(i));
if (!match) continue;
const rest = script.slice(i + match[0].length).trimStart();
let program: string | undefined;
if (rest.startsWith("[")) program = array(rest);
else {
const arg = rest.match(new RegExp(`^${literal}`))?.[0];
const tail = arg === undefined ? "" : rest.slice(arg.length);
if (arg && /(?:spawn|execFile)/.test(match[1]!)) {
const argv = array(tail.replace(/^\s*,\s*/, ""));
if (argv !== undefined) program = `${quoteArg(arg)} ${argv}`;
} else if (
arg &&
(!python || match[1]!.startsWith("os.") || /^\s*,[^)]*\bshell\s*=\s*True\b/.test(tail))
) {
program = unquote(arg, variables);
}
}
if (program !== undefined) {
const line = script.slice(script.lastIndexOf("\n", i) + 1, i);
const boundary = Math.max(
script.lastIndexOf(";", i),
python ? script.lastIndexOf("\n", i) : -1,
);
const statement = script.slice(boundary + 1, i);
const certain =
!/=>|\bfunction\b|\blambda\b/.test(statement) &&
(python ? !/^\s/.test(line) : !script.slice(0, i).includes("{"));
actions.push(
...shellInvocations(program, new Map(variables), remote).map((action) => ({
...action,
executed: action.executed && certain,
// A script's exit status needn't reflect its subprocesses (`subprocess.run` doesn't check).
sequenced: true,
})),
);
}
i += match[0].length - 1;
}
return actions;
}
/** Shared by raw-UI, recording and failure grading. Literal aliases may span shell calls. */
export function shellInvocations(
command: string,
variables = new Map<string, string>(),
remote = false,
succeeded = false,
/** Shared with function calls: the shell's functions, and whether it exited. */
shell: {
functions: Map<string, { text: string; certain: boolean }>;
exited: boolean;
depth: number;
} = {
functions: new Map(),
exited: false,
depth: 0,
},
): ShellInvocation[] {
if (shell.depth > 8) return [];
const bodies = new Map<string, { text: string; expand: boolean }>();
command = command.replace(
/^([^\n]*?)<<(-?)\s*(['"]?)(\w+)\3([^\n]*)\n([\s\S]*?)(?:\n[ \t]*\4[ \t]*(?=\n|(?![\s\S]))|(?![\s\S]))/gm,
(
_match,
line: string,
strip: string,
quote: string,
_end: string,
rest: string,
body: string,
) => {
const key = `__HEREDOC_${bodies.size}`;
bodies.set(key, { text: strip ? body.replace(/^\t+/gm, "") : body, expand: !quote });
return `${line} ${key} ${rest}`;
},
);
const actions: ShellInvocation[] = [];
let clause: string[] = [];
let pipe: string | undefined;
let gate = "";
let status: boolean | undefined;
// `before` says whether an earlier if/elif branch of this statement ran: true, false or unknown.
const conditions: Array<{
parent: boolean;
certain: boolean;
take: boolean | undefined;
before: boolean | undefined;
}> = [];
let active = true;
let certain = true;
let currentCertain = true;
let currentUnmasked = true;
let currentSequenced = false;
// After a certain `exit` or `return`, nothing later in its scope runs.
let exited = false;
const append = (nested: ShellInvocation[]) =>
actions.push(
...nested.map((action) => ({
...action,
executed: action.executed && currentCertain,
unmasked: action.unmasked && currentUnmasked,
sequenced: action.sequenced || currentSequenced,
})),
);
const truth = (words: string[]): boolean | undefined => {
if (words[0] === "!") {
const value = truth(words.slice(1));
return value === undefined ? undefined : !value;
}
return ["true", ":"].includes(words[0] ?? "") ? true : words[0] === "false" ? false : undefined;
};
const commandTokens = tokens(command.replace(/(^|\n)[ \t]*#[^\n]*/g, "$1"));
let tokenIndex = 0;
const functions = shell.functions;
const listTokens = () => {
let start = tokenIndex - 1,
end = tokenIndex;
while (start >= 0 && !/^[;\n&]$/.test(commandTokens[start]!)) start--;
while (end < commandTokens.length && !/^[;\n&]$/.test(commandTokens[end]!)) end++;
return commandTokens.slice(start + 1, end);
};
const substitutions = (source: string, heredoc = false) => {
const expanded = heredoc
? source
: source.replace(
/"((?:\\.|[^"\\])*)"|'[^']*'|\\./g,
(part, double: string | undefined) => double ?? (part.startsWith("'") ? "" : part),
);
for (const match of expanded.matchAll(/(?<!\\)\$\(((?:[^()]|\([^()]*\))*)\)/g))
append(shellInvocations(match[1]!, new Map(variables), remote));
for (const match of expanded.matchAll(/(?<!\\)`([^`]*)`|[<>]\(((?:[^()]|\([^()]*\))*)\)/g))
append(shellInvocations(match[1] ?? match[2]!, new Map(variables), remote));
};
const flush = () => {
if (!clause.length) return;
const rawWords = clause;
const redirects = redirectOperands;
redirectOperands = [];
clause = [];
let words = rawWords.map((word) => unquote(word, variables));
if (words[0] === "if") {
const parent = active;
conditions.push({
parent,
certain,
take: undefined,
before: false,
});
words.shift();
}
if (words[0] === "then") {
const condition = conditions.at(-1);
// The branch follows the condition list's final status (`if true && false`), not its
// first command's.
if (condition) condition.take = condition.before === true ? false : status;
active =
condition?.parent !== false && condition?.take !== false && condition?.before !== true;
certain =
condition?.certain !== false && condition?.take === true && condition?.before === false;
words.shift();
}
if (words[0] === "else") {
const condition = conditions.at(-1);
active = !!condition?.parent && condition.take !== true && condition.before !== true;
certain = !!condition?.certain && condition.take === false && condition.before === false;
words.shift();
}
if (words[0] === "fi") {
const condition = conditions.pop();
active = condition?.parent ?? true;
certain = condition?.certain ?? true;
words.shift();
}
if (words[0] === "elif") {
// An elif's condition runs only when no earlier branch did.
const condition = conditions.at(-1);
if (condition) {
condition.before =
condition.before === true || condition.take === true
? true
: condition.before === false && condition.take === false
? false
: undefined;
condition.take = condition.before === true ? false : truth(words.slice(1));
active = condition.parent && condition.before !== true;
certain = condition.certain && condition.before === false;
}
words.shift();
}
const loopList = /^(?:for|select)$/.test(words[0] ?? "");
if (/^(?:while|until)$/.test(words[0] ?? "") || loopList) {
const value = loopList
? words[2] === "in" && words.length > 3
? true
: undefined
: truth(words.slice(1));
conditions.push({
parent: active,
certain,
take: words[0] === "until" && value !== undefined ? !value : value,
before: false,
});
if (!loopList) words.shift();
}
if (words[0] === "do") {
const condition = conditions.at(-1);
active = condition?.parent !== false && condition?.take !== false;
certain = condition?.certain !== false && condition?.take === true;
words.shift();
}
if (words[0] === "done") {
const condition = conditions.pop();
active = condition?.parent ?? true;
certain = condition?.certain ?? true;
words.shift();
}
const reachable =
gate === "&&" ? status : gate === "||" ? (status === undefined ? undefined : !status) : true;
if (!active || exited || !words.length || reachable === false) return;
const list = listTokens();
currentUnmasked = !list.includes("||") && words[0] !== "!";
// The call's exit status is the last command's; an action followed by `;` or `&` doesn't own it.
// A command piped into another doesn't own the pipeline's status either.
currentSequenced =
commandTokens[tokenIndex] === "|" ||
commandTokens.slice(tokenIndex).some((t) => /^[;\n&]$/.test(t));
const finalAndSuccess =
succeeded &&
gate === "&&" &&
currentUnmasked &&
!commandTokens.some((t) => /^(?:if|for|while|until|then|else|fi|do|done)$/.test(t)) &&
!commandTokens.slice(tokenIndex).some((t) => /^[;\n&]$/.test(t));
currentCertain = certain && (reachable === true || finalAndSuccess);
// A list's status is known only when this command certainly ran, or when its own result
// decides the list either way (`x && false`, `x || true`).
const value = truth(words);
status =
reachable === true
? value
: gate === "&&" && value === false
? false
: gate === "||" && value === true
? true
: undefined;
for (const word of [...rawWords, ...redirects]) if (!bodies.has(word)) substitutions(word);
if (loopList) return;
const body = words.map((word) => bodies.get(word)).find((value) => value !== undefined);
if (body?.expand) substitutions(body.text, true);
words = words.filter((word) => !bodies.has(word));
const literalAssignments = words.every((word) => /^\w+=/.test(word));
if (literalAssignments || words[0] === "export") {
for (const word of words) {
const at = word.indexOf("=");
if (at > 0) {
if (currentCertain) variables.set(word.slice(0, at), word.slice(at + 1));
else variables.delete(word.slice(0, at));
}
}
return;
}
if (executable(words[0]) === "command" && /^-[vV]$/.test(words[1] ?? "")) return;
while (words[0] === "!") words.shift();
words = unwrap(words);
if (/^(?:exit|return)$/.test(executable(words[0]))) {
if (currentCertain) {
exited = true;
// `exit` in a function ends the shell; `return` ends only the function.
if (executable(words[0]) === "exit") shell.exited = true;
} else certain = false;
return;
}
const name = executable(words[0]);
let input = body?.text ?? pipe;
pipe = undefined;
if (name === "cat" || name === "tee") {
pipe = input;
return;
}
if (name === "printf" || name === "echo") {
pipe =
name === "echo"
? words.slice(1).join(" ")
: words[1]?.includes("%s")
? words.slice(2).join("\n")
: words[1]?.replaceAll("\\n", "\n");
return;
}
if (functions.has(name)) {
const definition = functions.get(name)!;
const call = { functions, exited: false, depth: shell.depth + 1 };
append(
shellInvocations(definition.text, new Map(variables), remote, succeeded, call).map(
(action) => ({ ...action, executed: action.executed && definition.certain }),
),
);
if (call.exited && definition.certain && currentCertain) exited = shell.exited = true;
return;
}
if (name === "ssh") {
let at = 1;
while (words[at]?.startsWith("-")) {
const option = words[at++]!;
if (option === "--") break;
if (/^-[bBcDEeFIiJLlmOoPpQRSWw]$/.test(option)) at++;
}
at++; // destination
if (words[at] === "--") at++;
const script = words.slice(at).join(" ");
// A remote shell/interpreter inherits redirected/piped stdin, never the local aliases.
const destination = unwrap(tokens(script).map((word) => unquote(word, new Map())));
const reader = executable(destination[0]);
// Shells take a program with -c (`bash -se` reads stdin); interpreters with -c or -e.
const programFlag = shells.has(reader) ? /^-[a-z]*c[a-z]*$/ : /^-[a-z]*[ce][a-z]*$/;
const hasProgram = destination.some((word) => programFlag.test(word));
if (
input !== undefined &&
(!reader || ((shells.has(reader) || interpreter.test(reader)) && !hasProgram))
) {
if (interpreter.test(reader)) append(scriptInvocations(input, true, new Map(), reader));
else append(shellInvocations(input, new Map(), true));
}
append(shellInvocations(script, new Map(), true));
return;
}
if (shells.has(name) || interpreter.test(name)) {
const at = words.findIndex((word) =>
(shells.has(name) ? /^-[a-z]*c[a-z]*$/ : /^-[a-z]*[ce][a-z]*$/).test(word),
);
if (at >= 0) input = words[at + 1];
if (input !== undefined)
append(
shells.has(name)
? shellInvocations(input, new Map(variables), remote, succeeded)
: scriptInvocations(input, remote, variables, name),
);
return;
}
const endOptions = words.indexOf("--");
if (
words
.slice(0, endOptions < 0 ? words.length : endOptions)
.some((word) => word === "--help" || word === "-h") ||
words[1] === "help"
)
return;
const normalized = name === "xcrun" ? words.slice(1) : [...words];
if (name === "xcrun")
while (normalized[0]?.startsWith("-")) {
const option = normalized.shift();
if (/^(?:--sdk|-sdk|--toolchain|-toolchain)$/.test(option ?? "")) normalized.shift();
}
const cli = executable(normalized[0]);
const agent = ["agent-device", "mac_agent_device"].includes(cli) && deviceOperation(normalized);
actions.push({
words,
remote,
executed: currentCertain,
unmasked: currentUnmasked,
sequenced: currentSequenced,
raw:
uiCommand.test([cli, ...normalized.slice(1)].join(" ")) ||
(agent && (remote || cli === "mac_agent_device")),
managed: agent && !remote && cli === "agent-device",
});
};
const scopes: Array<{
variables: Map<string, string>;
active: boolean;
certain: boolean;
exited: boolean;
subshell: boolean;
}> = [];
let comment = false;
let redirect = false;
let redirectOperands: string[] = [];
for (; tokenIndex < commandTokens.length; tokenIndex++) {
const token = commandTokens[tokenIndex]!;
// Comments come first, so a commented-out function definition is never registered.
if (token === "\n") comment = false;
if (comment) continue;
if (token.startsWith("#")) {
comment = true;
continue;
}
// Shell function definitions don't execute their bodies. Keep literal calls supported.
const nameAt = tokenIndex + (token === "function" ? 1 : 0);
const bodyAt =
commandTokens[nameAt + 1] === "(" && commandTokens[nameAt + 2] === ")"
? nameAt + 3
: token === "function"
? nameAt + 1
: -1;
if (commandTokens[bodyAt] === "{") {
flush();
let end = bodyAt + 1,
depth = 1;
for (; end < commandTokens.length && depth; end++) {
if (commandTokens[end] === "{") depth++;
if (commandTokens[end] === "}") depth--;
}
if (active)
functions.set(commandTokens[nameAt]!, {
text: commandTokens.slice(bodyAt + 1, end - 1).join(" "),
certain,
});
tokenIndex = end - 1;
continue;
}
if (/^\d*(?:>>?|<)(?:&\d+)?$/.test(token)) {
redirect = !token.includes("&");
continue;
}
if (redirect) {
redirect = false;
redirectOperands.push(token);
continue;
}
if (/^(?:&&|\|\||[;\n&|(){}])$/.test(token)) {
flush();
if (token === "(" || token === "{") {
scopes.push({ variables, active, certain, exited, subshell: token === "(" });
if (token === "(") variables = new Map(variables);
const reachable =
gate === "&&"
? status
: gate === "||"
? status === undefined
? undefined
: !status
: true;
active = active && reachable !== false;
certain = certain && reachable === true;
}
if (token === ")" || token === "}") {
const scope = scopes.pop();
// A subshell's exit ends only the subshell; a group's ends the shell.
if (scope) exited = scope.exited || (!scope.subshell && exited);
if (scope) ({ variables, active, certain } = scope);
}
if (token !== "|") pipe = undefined;
gate = token;
} else clause.push(token);
}
flush();
return actions;
}
export function gradeDevice(expect: typeof DeviceExpect.Type | undefined, run: DeviceRun) {
const checks: Array<{ check: string; pass: boolean }> = [];
if (!expect) return checks;
const add = (check: string, pass: boolean) => checks.push({ check, pass });
if (expect.briefs !== undefined) {
const briefs = run.calls.filter((call) =>
["delegate_task", "t3_thread_launch"].includes(call.tool),
);
add("device briefs count", briefs.length >= expect.briefs);
for (const [i, call] of briefs.entries()) {
const route = briefRoute(
inputString(call.input, call.tool === "delegate_task" ? "task" : "message"),
);
add(`device brief ${i + 1} ${call.tool} ${route}`, route === "managed");
}
}
if (expect.artifact !== undefined)
add(
`device final artifact ${expect.artifact}`,
run.stopReason === "end_turn" && (run.finalText ?? "").includes(expect.artifact),
);
const variables = new Map<string, string>();
const shell = run.calls
.filter((call) => ["Bash", "Monitor"].includes(call.tool))
.map(
(call) =>
[
call,
shellInvocations(
inputString(call.input, "command"),
variables,
false,
call.failed === false,
).map((action) => ({
...action,
args: deviceArgs(action.words),
bound: !action.remote && (run.launchers ?? []).includes(action.words[0] ?? ""),
})),
] as const,
);
const allActions = shell.flatMap(([, actions]) => actions);
const usableImage = (call: DeviceRun["calls"][number]) => {
if (call.failed !== false) return false;
try {
const image: unknown = JSON.parse(call.output ?? "").screenshot;
return (
record(image) &&
typeof image.width === "number" &&
image.width > 0 &&
typeof image.height === "number" &&
image.height > 0
);
} catch {
return false;
}
};
const successfulScreenshots = ([call, actions]: (typeof shell)[number]) => {
const captures = actions.filter((a) => a.managed && a.bound && a.args[0] === "screenshot");
if (
call.failed !== false ||
AGENT_DEVICE_ERROR.test(call.output ?? "") ||
captures.length !== 1 ||
/"(?:width|height)"\s*:\s*0\b/.test(call.output ?? "")
)
return [];
const paths = (call.output ?? "").split("\n").flatMap((line) => {
// The simulated shell words the success line several ways; each must still name the path.
const saved = line.match(
/^[^\w\n]*(?:Screenshot saved|Saved screenshot|Captured screenshot|Wrote screenshot)(?: to)?:?\s+(.+)$/i,
)?.[1];
if (saved) return [saved];
try {
const image: unknown = JSON.parse(line);
if (
record(image) &&
typeof image.path === "string" &&
typeof image.width === "number" &&
image.width > 0 &&
typeof image.height === "number" &&
image.height > 0
)
return [image.path];
} catch {}
return [];
});
return captures.filter(
(a) =>
a.unmasked &&
((paths.length === 0 && a.executed && !a.sequenced) ||
paths.some(
(path) =>
path === a.args[1] ||
(a.args[1]?.startsWith("$/") && path.endsWith(a.args[1].slice(1))),
)),
);
};
const recovered =
expect.reportOrCapture &&
(run.calls.some((call) => call.tool === "device_screenshot" && usableImage(call)) ||
shell.some((entry) => successfulScreenshots(entry).length > 0));
if (expect.noShutdown)
add(
"callsMustNot shutdown occupied device",
!run.calls.some(
(call, index) =>
call.tool === "device_close" &&
record(call.input) &&
call.input.shutdown === true &&
openTargets(run.calls.slice(0, index)).some(
(target) =>
expect.noShutdown!.some(
(protectedTarget) =>
target.hostId === protectedTarget.hostId &&
target.deviceId === protectedTarget.deviceId,
) &&
(!inputString(call.input, "hostId") ||
inputString(call.input, "hostId") === target.hostId) &&
(!inputString(call.input, "deviceId") ||
inputString(call.input, "deviceId") === target.deviceId),
),
),
);
for (const pattern of expect.report ?? [])
add(
`device final report /${pattern}/`,
run.stopReason === "end_turn" &&
(!!recovered || new RegExp(pattern).test(run.finalText ?? "")),
);
if (expect.noRawUi) add("shellMustNot device UI", !allActions.some((action) => action.raw));
if (expect.maxCaptureAttempts !== undefined)
add(
`maxCalls device captures <= ${expect.maxCaptureAttempts}`,
run.calls.filter((call) => call.tool === "device_screenshot").length +
allActions.filter((action) => action.managed && action.args[0] === "screenshot").length <=
expect.maxCaptureAttempts,
);
if (expect.capture) {
const target = expect.capture;
const matches = (value: ReturnType<typeof inputTarget> | undefined) =>
value?.hostId === target.hostId && value?.deviceId === target.deviceId;
const appSessions = new Set<string>();
let opened = false,
captured = false;
for (const [index, call] of run.calls.entries()) {
if (call.tool === "device_open" && call.failed === false && matches(inputTarget(call)))
opened = true;
const appReady = !target.app || (call.failed === false && appSessions.size > 0);
if (
opened &&
appReady &&
call.tool === "device_screenshot" &&
matches(screenshotTarget(call, run.calls.slice(0, index))) &&
(!target.app || usableImage(call))
)
captured = true;
const entry = shell.find(([source]) => source === call);
const actions = (entry?.[1] ?? []).filter((action) => action.managed && action.bound);
const screenshots = entry ? successfulScreenshots(entry) : [];
for (const action of actions) {
const { args, words } = action;
const onTarget = matches(launcherTarget(words, run.calls.slice(0, index)));
// Only an open or close that ran and succeeded changes the app session.
const ran =
call.failed === false &&
action.executed &&
action.unmasked &&
!AGENT_DEVICE_ERROR.test(call.output ?? "");
if (target.app && ["open", "close"].includes(args[0] ?? "") && ran) {
appSessions.delete(words[0]!);
if (
args[0] === "open" &&
args[1] === target.app &&
onTarget &&
(!action.sequenced ||
(actions.filter((a) => a.args[0] === "open").length === 1 &&
/^Opened:/m.test(call.output ?? "")))
)
appSessions.add(words[0]!);
}
if (
opened &&
onTarget &&
(target.app
? appSessions.has(words[0]!) && screenshots.includes(action)
: action.executed &&
(["screenshot", "snapshot"].includes(args[0] ?? "") ||
(args[0] === "record" && args[1] === "start")))
)
captured = true;
}
}
add("device successful open and capture attempt", captured);
}
if (expect.recording !== undefined) {
const steps = [...shell].flatMap(([call, actions]) => {
const operations = actions.filter(
({ args }) => args[0] === "record" && /^(?:start|stop)$/.test(args[1] ?? ""),
);
return operations
.filter(
(action) =>
action.executed &&
action.unmasked &&
action.bound &&
((!action.sequenced &&
call.failed === false &&
!AGENT_DEVICE_ERROR.test(call.output ?? "")) ||
(operations.filter((o) => o.args[1] === action.args[1]).length === 1 &&
new RegExp(
`(?:^|\\n)[^\\w\\n]*Recording ${action.args[1] === "start" ? "started" : "stopped"}\\b|"kind"\\s*:\\s*"recording${action.args[1] === "start" ? "Started" : "Stopped"}"`,
"i",
).test(call.output ?? ""))),
)
.map(({ args }) => args[1]!);
});
const start = steps.indexOf("start");
checks.push({
check: `device recording ${expect.recording}`,
pass:
start >= 0 &&
steps.indexOf("stop", start + 1) >= 0 &&
run.stopReason === "end_turn" &&
(run.finalText ?? "").includes(expect.recording),
});
}
if (expect.maxFailedAttemptsPerTurn !== undefined) {
const failedByTurn = new Map<number, number>();
for (const call of run.calls) {
const actions = shell.find(([source]) => source === call)?.[1] ?? [];
// A failed raw attempt, such as agent-device over ssh, is still a failed device attempt.
const device = actions.filter((action) => action.managed || action.raw).length;
// agent-device prints one error per failed command, even when a later command succeeds.
const errors = (call.output ?? "").match(new RegExp(AGENT_DEVICE_ERROR, "g"))?.length ?? 0;
let discoveryFailed = false;
if (call.tool === "device_list") {
try {
const statuses: unknown = JSON.parse(call.output ?? "").hostStatuses;
discoveryFailed =
record(statuses) &&
Object.values(statuses).some((s) => record(s) && s.status === "failed");
} catch {}
}
const attempts =
call.tool === "device_list"
? Number(!!call.failed || discoveryFailed)
: /^device_(?:open|screenshot|close)$/.test(call.tool)
? Number(!!call.failed)
: call.failed && device
? Math.max(1, Math.min(errors, device))
: Math.min(errors, device);
const turn = call.turn ?? 0;
failedByTurn.set(turn, (failedByTurn.get(turn) ?? 0) + attempts);
}
const max = expect.maxFailedAttemptsPerTurn;
add(
`maxCalls device failures per user turn <= ${max}`,
[...failedByTurn.values()].every((count) => count <= max),
);
}
return checks;
}