#!/usr/bin/env node
// @effect-diagnostics nodeBuiltinImport:off
/**
* Replays small fake threads on a Claude or Codex model to check T3's MCP tool use. A trimmed
* harness capture supplies the prompt and native tools; a cheap model plays the shell. See
* README.md for how to run it and add cases.
*/
import * as NodeChildProcess from "node:child_process";
import * as NodeCrypto from "node:crypto";
import * as NodeUtil from "node:util";
import * as NodeVM from "node:vm";
import * as NodeZlib from "node:zlib";
import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
import * as PlatformNodeCrypto from "@effect/platform-node/NodeCrypto";
import * as NodeServices from "@effect/platform-node/NodeServices";
import {
ComposerContextRecord,
DeviceOperationError,
DeviceSummary,
OrchestratorMcpTaskRole,
ProviderInstanceId,
} from "@t3tools/contracts";
import * as Cause from "effect/Cause";
import * as Context from "effect/Context";
import * as Config from "effect/Config";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Path from "effect/Path";
import * as Schedule from "effect/Schedule";
import * as Schema from "effect/Schema";
import * as Semaphore from "effect/Semaphore";
import { Tool } from "effect/ai";
import * as McpSchema from "effect/ai/McpSchema";
import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http";
import { taskPrompt } from "../../apps/server/src/mcp/delegatedTaskModes.ts";
import * as McpInvocationContext from "../../apps/server/src/mcp/McpInvocationContext.ts";
import { advertisedJsonSchema } from "../../apps/server/src/mcp/advertisedJsonSchema.ts";
import * as Contracts from "@t3tools/contracts";
import {
EnvironmentId,
ORCHESTRATOR_MCP_RUNNING_TASK_NEXT,
PullRequestState,
ThreadId,
} from "@t3tools/contracts";
import { projectComposerContextForProvider } from "@t3tools/shared/composerContextReferences";
import { formatThreadLink } from "@t3tools/shared/threadLinks";
import { buildCodexAdditionalContext } from "../../apps/server/src/provider/CodexDeveloperInstructions.ts";
import { makeSideChatContext } from "../../apps/server/src/orchestration-v2/ContextHandoffService.ts";
import { withInlineContext } from "../../apps/server/src/orchestration-v2/ContextHandoffDelivery.ts";
import { AttachmentToolkit } from "../../apps/server/src/mcp/toolkits/attachment/tools.ts";
import { DeviceToolkit } from "../../apps/server/src/mcp/toolkits/device/tools.ts";
import { EnvironmentToolkit } from "../../apps/server/src/mcp/toolkits/environment/tools.ts";
import { HtmlToolkit } from "../../apps/server/src/mcp/toolkits/html/tools.ts";
import { OrchestratorToolkit } from "../../apps/server/src/mcp/toolkits/orchestrator/tools.ts";
import { PreviewToolkit } from "../../apps/server/src/mcp/toolkits/preview/tools.ts";
import { PreviewControlsToolkit } from "../../apps/server/src/mcp/toolkits/previewControls/tools.ts";
import { ProjectToolkit } from "../../apps/server/src/mcp/toolkits/project/tools.ts";
import {
CHILD_PULL_REQUEST_REPORT_NEXT,
PullRequestsToolkit,
} from "../../apps/server/src/mcp/toolkits/pullRequests/tools.ts";
import { ThreadToolkit } from "../../apps/server/src/mcp/toolkits/thread/tools.ts";
import { WorktreeToolkit } from "../../apps/server/src/mcp/toolkits/worktree/tools.ts";
import {
CLAUDE_PROVIDER,
makeClaudeQueryOptions,
} from "../../apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts";
import { proxyConfig } from "../shared/proxy.ts";
import { changeRequestsCreatedBy } from "../../apps/server/src/orchestration-v2/PullRequestSyncReactor.ts";
import { DeviceExpect, gradeDevice } from "./device-grading.ts";
import {
AGENT_DEVICE_SHELL,
deviceFailure,
type DeviceTools,
isDeviceTool,
launcherHelp,
makeDeviceTools,
} from "./devices.ts";
import {
argvForm,
gradeSecret,
scriptSources,
SecretExpect,
secretDiagnostics,
} from "./secret-grading.ts";
/** Agent models by id, with the display name a harness gives them. */
const MODELS = {
"claude-sonnet-5-5": "Sonnet 5.5",
"claude-opus-5-5": "Opus 5.5",
"gpt-6.1-sol": "GPT-6.1 Sol",
"gpt-6-astra": "GPT-6 Astra",
} as const;
type Model = keyof typeof MODELS;
/** GPT models replay a Codex capture through CLIProxyAPI, the rest a Claude Code capture. */
const isCodex = (model: Model): model is Extract<Model, `gpt-${string}`> =>
model.startsWith("gpt-");
const EFFORTS = ["low", "medium", "high", "xhigh", "max"] as const;
// USD per million tokens. Cache writes use the 5-minute rate.
const PRICES = {
"claude-sonnet-5-5": { input: 2, output: 10, cacheWrite: 2.5, cacheRead: 0.2 },
"claude-haiku-5-5": { input: 0.1, output: 0.5, cacheWrite: 0.125, cacheRead: 0.01 },
"claude-opus-5-5": { input: 4, output: 20, cacheWrite: 5, cacheRead: 0.2 },
};
type PricedModel = keyof typeof PRICES;
/** GPT models that can play the shell through CLIProxyAPI, when no Anthropic key is at hand. */
const PROXY_SHELL_MODELS = ["gpt-6-luna", "gpt-6.1-terra"] as const;
type ShellModel = PricedModel | (typeof PROXY_SHELL_MODELS)[number];
const isPricedModel = (model: ShellModel): model is PricedModel => model in PRICES;
const T3_PREFIX = "mcp__t3-code__";
const RUNTIME_MARKER = "<runtime_info>";
const TOOL_LOADED = "Tool loaded.";
class EvalToolCallError extends Schema.TaggedError<EvalToolCallError>()("EvalToolCallError", {
tool: Schema.String,
cause: Schema.Defect(),
}) {
override get message(): string {
return `The ${this.tool} tool call failed.`;
}
}
const DEFERRED_LIST =
/(The following deferred tools are now available via ToolSearch\.[^\n]*\n)((?:[^\n]+\n)*)/;
const MODEL_LINE = /You are powered by the model named .+?\. The exact model ID is [\w.-]+\./;
// API keys can be billed differently for requests that open with Claude Code's identity line,
// so every copy of it becomes a neutral line. The rest of the captured prompt is unchanged.
const IDENTITY =
/You are Claude Code, Anthropic's official CLI for Claude(?:, running within the Claude Agent SDK)?\./g;
const NEUTRAL_IDENTITY = "You are an AI coding agent running in the user's terminal.";
type Block = { readonly type: string; readonly [key: string]: unknown };
type ApiMessage = { readonly role: "user" | "assistant"; readonly content: ReadonlyArray<Block> };
type ApiTool = { readonly name: string; readonly [key: string]: unknown };
type Usage = Record<keyof (typeof PRICES)[PricedModel], number>;
type Call = {
readonly step: number;
readonly tool: string;
readonly input: unknown;
/** The tool_use id, when the call was a tool use of its own. */
readonly id?: string;
/** Once answered: whether the agent saw an error, and the result text it saw. */
failed?: boolean;
output?: string;
};
export type SampleRun = {
readonly calls: ReadonlyArray<Call>;
text: string;
stopReason: string;
/** The agent's final answer: its last response's text, if it ended its turn. */
readonly finalText?: string;
/** PRs linked to the thread when the sample ends, by the agent or by T3. */
readonly linked?: ReadonlyArray<string>;
/** The `agent-device` commands T3's device_open returned, prior turns included. */
readonly launchers?: ReadonlyArray<string>;
};
const Rest = [Schema.Record(Schema.String, Schema.Unknown)] as const;
const PriorToolCall = Schema.Struct({
tool: Schema.String,
input: Schema.Unknown,
/** For a device tool, `"default"` runs the call on the world's devices for T3's result. */
result: Schema.Unknown,
/** Repository-relative file whose current contents supply this tool result. */
resultFile: Schema.optional(Schema.String),
});
/**
* One expected call. `args` match partially: `~text` is a case-insensitive substring,
* `"<=N"` / `">=N"` bound a number, `null` matches a missing key, and keys may be dotted paths.
*/
const McpToolUseCall = Schema.Struct({
tool: Schema.String,
args: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
});
type McpToolUseCall = typeof McpToolUseCall.Type;
/** Grading rules. Tool names are bare: `delegate_task`, `Bash`, `Agent`. */
const McpToolUseExpect = Schema.Struct({
/** Each entry needs one matching call. */
calls: Schema.optional(Schema.Array(McpToolUseCall)),
/** Each entry matches a different call, and no call matches two entries; tool names are exact. */
distinctCalls: Schema.optional(Schema.Array(McpToolUseCall)),
/** No call may match any entry. */
callsMustNot: Schema.optional(Schema.Array(McpToolUseCall)),
notCalls: Schema.optional(Schema.Array(Schema.String)),
/** First calls of these tools happen in this order. */
order: Schema.optional(Schema.Array(Schema.String)),
minCalls: Schema.optional(Schema.Record(Schema.String, Schema.Number)),
maxCalls: Schema.optional(Schema.Record(Schema.String, Schema.Number)),
/**
* One `args`-style value per new top-level thread the agent asks for, by `t3_thread_launch` or a
* `create_threads` entry. Each matches a different thread's request; no other thread is asked for.
*/
newThreads: Schema.optional(Schema.Array(Schema.Unknown)),
/** The last step with tool calls includes this tool, then the turn ends. */
endsTurnAfter: Schema.optional(Schema.String),
/**
* Regexes that no shell command may match. In a case with `secret` checks, a command also carries
* the source of the scripts it runs that the agent wrote earlier in the sample.
*/
shellMustNot: Schema.optional(Schema.Array(Schema.String)),
/** Regexes that some shell command must match, each. */
shellMust: Schema.optional(Schema.Array(Schema.String)),
/** Case-sensitive regexes on the agent's visible text. */
textMust: Schema.optional(Schema.Array(Schema.String)),
textMustNot: Schema.optional(Schema.Array(Schema.String)),
/** Case-sensitive regexes on the final answer: the last response's text, once the turn ended. */
finalTextMust: Schema.optional(Schema.Array(Schema.String)),
/** Each entry matches a PR URL linked at the end, as `args` values match. */
linked: Schema.optional(Schema.Array(Schema.String)),
/** The PR numbers linked at the end, exactly. */
linkedNumbers: Schema.optional(Schema.Array(Schema.Number)),
/** Device manager checks: child briefs, simulator UI over the shell, failures, recordings. */
device: Schema.optional(DeviceExpect),
/**
* Secret checks: chat asks in text and question tools, where fixture values went, and how the
* files request_secret returned were read. See `SecretExpect` in secret-grading.ts.
*/
secret: Schema.optional(SecretExpect),
});
/**
* The agent's machine, as the shell simulator and the gitStatus reminder see it. Only these keys
* are typed; others (files, diff, prs, pages) are free-form facts for the simulator.
*/
const McpToolUseWorld = Schema.StructWithRest(
Schema.Struct({
/** GitHub `owner/name`. Defaults to `acme/app`. */
repo: Schema.optional(Schema.String),
branch: Schema.optional(Schema.String),
/** `git status --short` lines. */
status: Schema.optional(Schema.String),
/** Newest first. On a branch other than main, these are the commits main lacks. */
commits: Schema.optional(
Schema.Array(
Schema.StructWithRest(Schema.Struct({ sha: Schema.String, subject: Schema.String }), Rest),
),
),
/** Simulators and emulators. With them, T3's device tools answer from this world. */
devices: Schema.optional(Schema.Array(DeviceSummary)),
/** Labels of the SSH device hosts in `devices`, by id. The local host always exists. */
deviceHosts: Schema.optional(Schema.Record(Schema.String, Schema.String)),
/**
* agent-device sessions that prior turns started, by device id, with the app `open` put in
* each. Otherwise a device's session starts at its launcher's first `open` in the sample.
*/
agentDeviceSessions: Schema.optional(
Schema.Record(Schema.String, Schema.Struct({ app: Schema.optional(Schema.String) })),
),
/**
* Hosts whose hub fails to list devices, by id, with the error, such as F1's
* `{"_tag": "DeviceOperationError", "operation": "list", "reason": "request_failed", "cause": null}`.
* T3 marks each one failed on every call and lists none of its devices.
*/
deviceDiscoveryFailures: Schema.optional(Schema.Record(Schema.String, DeviceOperationError)),
/** false: agent device access is off. T3 denies every device call and drops its device text. */
deviceAgentAccess: Schema.optional(Schema.Boolean),
/** Every capture reaches T3 with adb screencap's warning ahead of the PNG: T3 reports 0×0. */
deviceInvalidCapture: Schema.optional(Schema.Boolean),
/** Devices another thread's agent-device session holds, by device id: that thread's id. */
deviceOwners: Schema.optional(Schema.Record(Schema.String, Schema.String)),
/** PRs already linked to the thread, with the state T3 last saw. */
linkedPullRequests: Schema.optional(
Schema.Array(
Schema.Struct({ number: Schema.Number, state: PullRequestState, title: Schema.String }),
),
),
}),
Rest,
);
/** One fake thread, one JSON object per line in a case file. */
export const McpToolUseCase = Schema.Struct({
id: Schema.String,
area: Schema.String,
kind: Schema.Literals(["bad", "guard", "control"]),
pattern: Schema.String,
experience: Schema.String,
rule: Schema.optional(Schema.String),
privacy: Schema.optional(Schema.Literals(["commit-safe", "private"])),
setup: Schema.Struct({
role: Schema.Literals(["top", "delegated-child", "native-subagent"]),
/** Runs only on this harness, for cases whose right answer depends on it. */
harness: Schema.optional(Schema.Literals(["claude", "codex"])),
/** The Linked environments beta is on, so T3 lists its link tools. */
linkedEnvironments: Schema.optional(Schema.Boolean),
/** delegate_task `role` for a delegated child. T3 builds the child's first message from it. */
delegateRole: Schema.optional(OrchestratorMcpTaskRole),
context: Schema.optional(Schema.String),
priorTurns: Schema.optional(
Schema.Array(
Schema.Struct({
role: Schema.Literals(["user", "assistant"]),
text: Schema.optional(Schema.String),
toolCalls: Schema.optional(Schema.Array(PriorToolCall)),
}),
),
),
world: Schema.optional(McpToolUseWorld),
/**
* Canned tool results by bare name, T3 or native. `"default"` uses the runner's result. For a
* device tool, a T3 `DeviceError` (`{"_tag": ..., "cause": null}`) fails it inside T3.
*/
mocks: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
/** Regexes on one part of a shell command. A match wins over the shell simulator. */
shellMocks: Schema.optional(
Schema.Array(Schema.Struct({ match: Schema.String, output: Schema.String })),
),
/** The thread is a side chat's first turn: T3 puts the parent's digest ahead of `prompt`. */
sideChat: Schema.optional(
Schema.Struct({ parentThreadId: Schema.String, digest: Schema.Array(Schema.String) }),
),
}),
/** Empty to resume a top-level thread without side-chat context after its last tool result. */
prompt: Schema.String,
/** Records for the context chips `prompt` links, projected as T3 sends them. */
promptContext: Schema.optional(Schema.Struct({ records: Schema.Array(ComposerContextRecord) })),
expect: McpToolUseExpect,
/** Model calls this case may need. Overrides `--max-steps`. */
maxSteps: Schema.optional(Schema.Number),
source: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
notes: Schema.optional(Schema.String),
});
export type McpToolUseCase = typeof McpToolUseCase.Type;
const AnyBlock = Schema.StructWithRest(
Schema.Struct({ type: Schema.String, text: Schema.optional(Schema.String) }),
Rest,
);
const Content = Schema.Union([Schema.String, Schema.Array(AnyBlock)]);
const MessagesRequest = Schema.Struct({
system: Schema.optional(Content),
messages: Schema.Array(Schema.Struct({ role: Schema.String, content: Content })),
tools: Schema.Array(Schema.StructWithRest(Schema.Struct({ name: Schema.String }), Rest)),
});
// A CLIProxyAPI capture keeps the Anthropic body at request.provider_native.
const Capture = Schema.Struct({ request: Schema.Struct({ provider_native: MessagesRequest }) });
const ResponsesRequest = Schema.StructWithRest(
Schema.Struct({
input: Schema.Array(AnyBlock),
reasoning: Schema.StructWithRest(Schema.Struct({ effort: Schema.String }), Rest),
}),
Rest,
);
const CodexCapture = Schema.Struct({
request: Schema.Struct({ provider_native: ResponsesRequest }),
});
const ToolUse = Schema.Struct({
type: Schema.Literal("tool_use"),
id: Schema.String,
name: Schema.String,
input: Schema.Unknown,
});
const isToolUse = Schema.is(ToolUse);
const isText = Schema.is(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String }));
const isRecord = (value: unknown): value is Readonly<Record<string, unknown>> =>
typeof value === "object" && value !== null && !Array.isArray(value);
const Count = Schema.optional(Schema.NullOr(Schema.Number));
const ApiResponse = Schema.Struct({
content: Schema.Array(AnyBlock),
stop_reason: Schema.NullOr(Schema.String),
usage: Schema.Struct({
input_tokens: Schema.Number,
output_tokens: Schema.Number,
cache_creation_input_tokens: Count,
cache_read_input_tokens: Count,
}),
});
const emptyUsage = (): Usage => ({ input: 0, output: 0, cacheWrite: 0, cacheRead: 0 });
const addUsage = (total: Usage, usage: typeof ApiResponse.Type.usage) => {
total.input += usage.input_tokens;
total.output += usage.output_tokens;
total.cacheWrite += usage.cache_creation_input_tokens ?? 0;
total.cacheRead += usage.cache_read_input_tokens ?? 0;
};
const text = (value: string): Block => ({ type: "text", text: value });
const reminder = (value: string) => text(`<system-reminder>\n${value}\n</system-reminder>`);
const valueText = (value: unknown) =>
typeof value === "string" ? value : (JSON.stringify(value) ?? "");
const toolResult = (id: string, content: unknown): Block => ({
type: "tool_result",
tool_use_id: id,
content: valueText(content),
});
const texts = (content: typeof Content.Type) =>
typeof content === "string" ? [content] : content.flatMap((block) => block.text ?? []);
const isT3Name = (name: string) => name.startsWith(T3_PREFIX) || name.startsWith("mcp__t3_code__");
const hasTool = (tools: ReadonlyArray<ApiTool>, name: string) =>
tools.some((tool) => tool.name === name);
/** Grading name: T3 tools lose their MCP prefix, native tools keep the part after the last `_`. */
const CODEX_NAMES: Readonly<Record<string, string>> = {
exec_command: "Bash",
shell_command: "Bash",
shell: "Bash",
spawn_agent: "Agent",
apply_patch: "Edit",
request_user_input: "AskUserQuestion",
request_user_input_async: "AskUserQuestion",
write_stdin: "write_stdin",
web__run: "WebFetch",
};
export const normalizeToolName = (name: string) => {
const bare = name.slice(name.lastIndexOf(".") + 1);
return isT3Name(bare)
? bare.slice(T3_PREFIX.length)
: (CODEX_NAMES[bare] ?? bare.slice(bare.lastIndexOf("_") + 1));
};
export const normalizeToolInput = (name: string, input: unknown) => {
const decode = t3InputDecoders.get(normalizeToolName(name));
if (decode !== undefined) return Option.getOrElse(decode(input), () => input);
return normalizeToolName(name) === "Bash" && isRecord(input)
? { ...input, command: input.command ?? input.cmd }
: typeof input === "string" && normalizeToolName(name) === "Edit"
? { patch: input }
: input;
};
// McpHttpServer advertises each tool's JSON schema, and Claude Code passes the MCP name,
// description, and schema through unchanged.
const t3SourceTools: ReadonlyArray<Tool.Any> = [
OrchestratorToolkit,
ThreadToolkit,
AttachmentToolkit,
ProjectToolkit,
EnvironmentToolkit,
PreviewControlsToolkit,
WorktreeToolkit,
PullRequestsToolkit,
PreviewToolkit,
DeviceToolkit,
HtmlToolkit,
].flatMap((toolkit): ReadonlyArray<Tool.Any> => Object.values(toolkit.tools));
// McpHttpServer decodes arguments with each tool's JSON codec, which drops undeclared keys.
const t3InputDecoders = new Map(
t3SourceTools.map((tool) => [
tool.name,
Schema.decodeUnknownOption(
Schema.toCodecJson(tool.parametersSchema as Schema.Codec<unknown, unknown>),
),
]),
);
const t3Tools: ReadonlyArray<ApiTool> = t3SourceTools.map((tool) => ({
name: `${T3_PREFIX}${tool.name}`,
description: Tool.getDescription(tool) ?? "",
input_schema: advertisedJsonSchema(tool),
}));
const enabledWhen = (predicate: () => boolean) =>
new Set(
t3SourceTools
.filter(
(tool) => Context.getOrUndefined(tool.annotations, McpSchema.EnabledWhen) === predicate,
)
.map((tool) => `${T3_PREFIX}${tool.name}`),
);
// Claude Code loads a tool whose `_meta` sets anthropic/alwaysLoad up front instead of deferring it.
const alwaysLoaded = new Set(
t3SourceTools
.filter((tool) => {
const meta = Context.getOrUndefined(tool.annotations, Tool.Meta);
return meta !== undefined && meta["anthropic/alwaysLoad"] === true;
})
.map((tool) => `${T3_PREFIX}${tool.name}`),
);
// Refs whose request_secret takes `value` save a secret the user already pasted, with no card.
export const savesPastedSecrets = t3Tools.some(
(tool) =>
tool.name === `${T3_PREFIX}request_secret` &&
isRecord(tool.input_schema) &&
isRecord(tool.input_schema.properties) &&
"value" in tool.input_schema.properties,
);
/** A request_secret call that hands T3 a pasted secret, on a ref that takes one. */
const pastedSecret = (name: string, input: Readonly<Record<string, unknown>>) =>
savesPastedSecrets && name === "request_secret" && typeof input.value === "string";
// T3 does not offer these to a delegated child, so its harness never lists them.
const hiddenFromSubagents = enabledWhen(McpInvocationContext.hiddenFromSubagents);
// The Linked environments beta starts off, and T3 lists its tools only while it is on.
const hiddenWhileLinksOff = enabledWhen(McpInvocationContext.hiddenWhileLinksOff);
const visibleT3Tools = (options: { readonly subagent?: boolean; readonly links?: boolean }) =>
t3Tools.filter(
(tool) =>
(options.links === true || !hiddenWhileLinksOff.has(tool.name)) &&
!(options.subagent === true && hiddenFromSubagents.has(tool.name)),
);
/**
* T3's Claude Code options for an eval thread: its system prompt append and its hooks. `device` says
* whether the thread holds the `device` capability, as agent device access decides.
*/
const claudeQueryOptions = (model: Model, device = true) => {
// Refs whose device text follows the capability read deviceToolsAvailable; older ones ignore it.
const input: Parameters<typeof makeClaudeQueryOptions>[0] & {
readonly deviceToolsAvailable?: boolean;
} = {
modelSelection: { instanceId: ProviderInstanceId.make(CLAUDE_PROVIDER), model },
nativeThreadId: "mcp-tool-use-eval",
resume: false,
cwd: null,
mcpServers: { "t3-code": { type: "http", url: "http://127.0.0.1/mcp" } },
deviceToolsAvailable: device,
};
return makeClaudeQueryOptions(input);
};
/** Text T3 appends to Claude Code's system prompt. Delegated children get the same text. */
export const t3SystemAppend = (model: Model, device = true) => {
const { systemPrompt } = claudeQueryOptions(model, device);
if (isRecord(systemPrompt) && systemPrompt.type === "preset" && systemPrompt.append) {
return systemPrompt.append;
}
throw new Error("makeClaudeQueryOptions no longer appends T3 text to a preset prompt.");
};
/**
* The context T3's SubagentStart hooks give a native subagent, which gets none of the append.
* Empty when T3 registers no such hook.
*/
export const t3SubagentContext = async (model: Model, device = true) => {
const hooks = (claudeQueryOptions(model, device).hooks?.SubagentStart ?? []).flatMap(
(m) => m.hooks,
);
const input = {
hook_event_name: "SubagentStart" as const,
session_id: "mcp-tool-use-eval",
transcript_path: "",
cwd: "",
agent_id: "mcp-tool-use-eval",
agent_type: "general-purpose",
};
const outputs = await Promise.all(
hooks.map((hook) => hook(input, undefined, { signal: AbortSignal.timeout(10_000) })),
);
return outputs
.flatMap((output) =>
"hookSpecificOutput" in output && output.hookSpecificOutput?.hookEventName === "SubagentStart"
? [output.hookSpecificOutput.additionalContext ?? ""]
: [],
)
.filter(Boolean)
.join("\n\n");
};
/**
* Size of the context T3 adds: the system text every Claude thread gets, and the tool
* descriptions and schemas (loaded on demand here, sent up front by harnesses without
* deferred tools). Tokens are estimated at 4 characters each.
*/
const t3ContextReport = (model: Model) => {
const size = (chars: number) => `${chars} chars (~${Math.round(chars / 4)} tokens)`;
const descriptions = t3Tools.reduce((sum, tool) => sum + valueText(tool.description).length, 0);
const schemas = JSON.stringify(t3Tools).length;
return [
`- T3 context: system text ${size((isCodex(model) ? JSON.stringify(buildCodexAdditionalContext({ model }, { browser: true, device: true })) : t3SystemAppend(model)).length)}`,
`tool descriptions ${size(descriptions)}`,
`full tool schemas ${size(schemas)} across ${t3Tools.length} tools.`,
].join(", ");
};
const stringProperty = (description: string) => ({ type: "string", description });
const NATIVE_SCHEMAS: Readonly<Record<string, Omit<ApiTool, "name">>> = {
WebFetch: {
description: "Fetches content from a URL and answers the prompt about it.",
input_schema: {
type: "object",
properties: {
url: stringProperty("The URL to fetch"),
prompt: stringProperty("What to extract"),
},
required: ["url", "prompt"],
},
},
WebSearch: {
description: "Searches the web and returns result titles, links, and snippets.",
input_schema: {
type: "object",
properties: { query: stringProperty("The search query") },
required: ["query"],
},
},
};
/** Rebuilds the parts of a captured request that every case shares. */
export const buildBase = (
template: typeof MessagesRequest.Type,
options: {
readonly model: Model;
readonly keepUserInstructions: boolean;
readonly subagent?: boolean;
readonly links?: boolean;
/** A native subagent starts from its task: T3 appends nothing to its system prompt. */
readonly nativeSubagent?: boolean;
/** false: the thread has no agent device access, so T3's text leaves devices out. */
readonly deviceAccess?: boolean;
},
) => {
const t3 = visibleT3Tools(options);
const firstAssistant = template.messages.findIndex((message) => message.role === "assistant");
const head = template.messages.slice(0, firstAssistant === -1 ? undefined : firstAssistant);
const systemTexts = [
template.system ?? [],
...head.flatMap((m) => (m.role === "system" ? [m.content] : [])),
]
.flatMap(texts)
.filter((value) => !value.startsWith("x-anthropic-billing-header:"));
const deferredList = DEFERRED_LIST.exec(systemTexts.join("\n"))?.[2];
if (systemTexts.filter((value) => value.includes(RUNTIME_MARKER)).length !== 1 || !deferredList) {
throw new Error(`The template needs one ${RUNTIME_MARKER} block and a deferred tool list.`);
}
const nativeDeferred = deferredList.split("\n").filter((name) => name && !isT3Name(name));
const names = [
...nativeDeferred,
...t3.filter((tool) => !alwaysLoaded.has(tool.name)).map((tool) => tool.name),
]
.toSorted()
.join("\n");
const { model } = options;
const append = options.nativeSubagent ? "" : t3SystemAppend(model, options.deviceAccess);
const modelLine = `You are powered by the model named ${MODELS[model]}. The exact model ID is ${model}.`;
const joined = systemTexts.join("\n");
// The capture's home directory names its user. Cases use `/home/user` paths instead.
const cwd = /Primary working directory: (\S+)/.exec(joined)?.[1] ?? "/home/user/app";
const home = /^\/(?:home|Users)\/[^/]+/.exec(cwd)?.[0];
const anonymize = (value: string) =>
home === undefined ? value : value.replaceAll(home, home.replace(/[^/]+$/, "user"));
const system = systemTexts.map((value) => {
// T3's append is the tail of its block: runtime info, PR linking, then orchestration.
const at = value.indexOf(RUNTIME_MARKER);
return anonymize(at === -1 ? value : value.slice(0, at) + append)
.replace(DEFERRED_LIST, (_, header: string) => `${header}${names}\n`)
.replace(MODEL_LINE, modelLine)
.replace(IDENTITY, NEUTRAL_IDENTITY);
});
const [first] = template.messages;
// The last block is the captured prompt.
const reminders = (first?.role === "user" ? texts(first.content).slice(0, -1) : []).filter(
(value) =>
options.keepUserInstructions || !value.includes("Codebase and user instructions are"),
);
// gitStatus names a real repo state. Each case gets one built from its fake world instead.
const gitStatusAt = reminders.findIndex((value) => value.includes("# gitStatus"));
// Claude Code sends no schema for tools it only names. Web tools get their usual one, the
// rest load open.
const open = { type: "object", additionalProperties: true };
const nativeSchema = (name: string) =>
NATIVE_SCHEMAS[name] ?? { description: name, input_schema: open };
return {
model,
system: system.map(text),
reminders: reminders.filter((_, i) => i !== gitStatusAt).map((value) => text(anonymize(value))),
gitStatusAt: gitStatusAt === -1 ? reminders.length : gitStatusAt,
/** The shell simulator's cwd and OS follow the template's environment section. */
environment: {
cwd: anonymize(cwd),
platform: /Platform: (\w+)/.exec(joined)?.[1] ?? "linux",
date: /Today's date is ([\d-]+)/.exec(reminders.join("\n"))?.[1],
},
nativeTools: [
...template.tools.filter((tool) => !isT3Name(tool.name)),
...t3.filter((tool) => alwaysLoaded.has(tool.name)),
],
/** Tools ToolSearch loads: T3 tools from source, and native tools the template only names. */
t3,
deferred: [...t3, ...nativeDeferred.map((name) => ({ name, ...nativeSchema(name) }))],
};
};
type Base = ReturnType<typeof buildBase> & {
readonly codex?: typeof ResponsesRequest.Type;
/** What T3's SubagentStart hooks give a native subagent. */
readonly subagentContext?: string;
};
/** Keep the captured harness prompt and tools; replace app-owned context and the real user. */
const buildCodexBase = (
template: typeof ResponsesRequest.Type,
options: {
readonly model: Model;
readonly keepUserInstructions: boolean;
readonly effort?: string | undefined;
readonly subagent?: boolean;
readonly links?: boolean;
/** false: the thread has no agent device access, so T3's text leaves devices out. */
readonly deviceAccess?: boolean;
},
): Base => {
const t3 = visibleT3Tools(options);
const contentText = (item: Block) =>
Array.isArray(item.content)
? item.content.map((block: Block) => valueText(block.text)).join("\n")
: valueText(item.content);
const joined = template.input.map(contentText).join("\n");
if (!joined.includes("<t3_code_runtime>"))
throw new Error("Codex template needs full T3 context.");
const cwd = /<cwd>(.*?)<\/cwd>/.exec(joined)?.[1] ?? "/home/user/app";
const home = /^\/(?:home|Users)\/[^/]+/.exec(cwd)?.[0];
const clean = (s: string) => (home ? s.replaceAll(home, home.replace(/[^/]+$/, "user")) : s);
const effort = options.effort ?? template.reasoning.effort;
const message = (value: string, role = "developer") => ({
type: "message",
role,
content: [{ type: "input_text", text: value }],
});
// The captured <collaboration_mode> message stays: Codex supplies its own mode text natively.
const input = template.input.flatMap((item) => {
const content = contentText(item);
if (content.includes("<t3_code_")) return [];
if (item.role === "user") {
const env = /<environment_context>[\s\S]*?<\/environment_context>/.exec(content)?.[0];
return env ? [message(clean(options.keepUserInstructions ? content : env), "user")] : [];
}
// Only harness messages and tools. A captured turn's answers and tool calls would leak into
// every case.
if (item.type !== "message" && item.type !== "additional_tools") return [];
if (item.role === "assistant") return [];
return [JSON.parse(clean(JSON.stringify(item))) as Block];
});
const context = buildCodexAdditionalContext(
{ model: options.model, reasoningEffort: effort },
{ browser: true, device: options.deviceAccess ?? true },
);
input.push(
...Object.entries(context).map(([key, entry]) => message(`<${key}>${entry.value}</${key}>`)),
);
const nativeTools = input
.flatMap((item) =>
item.type === "additional_tools" && Array.isArray(item.tools)
? item.tools.flatMap((ns: unknown) =>
isRecord(ns) && Array.isArray(ns.tools) ? ns.tools : [],
)
: [],
)
.filter((tool): tool is ApiTool => isRecord(tool) && typeof tool.name === "string");
if (!nativeTools.some((tool) => tool.name === "exec"))
throw new Error("Codex template needs code-mode exec.");
return {
model: options.model,
system: [],
reminders: [],
gitStatusAt: 0,
environment: {
cwd: clean(cwd),
platform: cwd.startsWith("/Users/") ? "darwin" : "linux",
date: /<current_date>(.*?)<\/current_date>/.exec(joined)?.[1],
},
nativeTools,
t3,
deferred: [...t3],
codex: { ...template, input, reasoning: { ...template.reasoning, effort } },
};
};
/** Replay prior calls in Codex's wire format, including Claude-shaped file operations. */
const codexCall = (name: string, input: unknown) => {
const args = isRecord(input) ? input : {};
if (name === "Bash")
return `text(await tools.exec_command(${JSON.stringify({ ...args, cmd: args.command })}));`;
if (["Read", "Write", "Edit"].includes(name)) {
const file = valueText(args.file_path);
const command =
name === "Read"
? `cat ${JSON.stringify(file)}`
: name === "Write"
? `cat > ${JSON.stringify(file)} <<'EVAL_EOF'\n${valueText(args.content)}\nEVAL_EOF`
: `python3 -c ${JSON.stringify(`from pathlib import Path; p=Path(${JSON.stringify(file)}); p.write_text(p.read_text().replace(${JSON.stringify(args.old_string)}, ${JSON.stringify(args.new_string)}))`)}`;
return `text(await tools.exec_command(${JSON.stringify({ cmd: command })}));`;
}
return `text(await tools.${name.replace("mcp__t3-code__", "mcp__t3_code__")}(${JSON.stringify(input)}));`;
};
const codexMessages = (messages: ReadonlyArray<ApiMessage>): Array<Block> =>
messages.flatMap((message) =>
message.content.map((block): Block =>
block.type === "tool_use"
? {
type: "custom_tool_call",
call_id: block.id,
name: "exec",
input: codexCall(valueText(block.name), block.input),
}
: block.type === "tool_result"
? {
type: "custom_tool_call_output",
call_id: block.tool_use_id,
output: valueText(block.content),
}
: {
type: "message",
role: message.role,
content: [
{
type: message.role === "user" ? "input_text" : "output_text",
text: valueText(block.text),
},
],
},
),
);
/** One tool call the shell simulator has answered, or one from the case's prior turns. */
type ShellEntry = { readonly tool: string; readonly input: unknown; readonly output: string };
/** PRs T3 links by itself because a shell command created them, as `gh pr create` does. */
const createdPullRequests = ({ tool, input, output }: ShellEntry) =>
tool === "Bash" && isRecord(input) && typeof input.command === "string"
? changeRequestsCreatedBy(input.command, output).map(({ url }) => url)
: [];
/**
* One sample's state. ToolSearch adds loaded schemas to `tools`, like Claude Code. `shell` is the
* history the simulator keeps consistent with; `linked`, `nextTask` and `threads` do the same for
* T3 results.
*/
type Sample = {
readonly testCase: McpToolUseCase;
readonly base: Base;
readonly tools: Array<ApiTool>;
readonly repo: string;
readonly shell: Array<ShellEntry>;
readonly shellUsage: Usage;
readonly linked: Set<string>;
readonly codeStore: Record<string, unknown>;
nextTask: number;
/** Threads the agent launched in this sample, by id, so reading one shows its own brief. */
readonly threads: Map<string, { readonly title: string; readonly brief: string }>;
/** Files T3's request_secret saved in this sample, in call order. */
readonly secretFiles: Array<string>;
secretRequests: number;
/** T3's device tools, once `withDevices` builds them for a world with devices. */
devices?: DeviceTools;
/** T3's results for prior device calls whose case result is "default". */
readonly priorResults: Map<unknown, string>;
};
const SHELL_TOOLS = new Set(["Bash", "Read", "Edit", "Write", "WebFetch"]);
export const makeSample = (testCase: McpToolUseCase, base: Base): Sample => {
const prior = testCase.setup.priorTurns ?? [];
const priorCalls = prior.flatMap((turn) => turn.toolCalls ?? []);
const repo = testCase.setup.world?.repo ?? "acme/app";
const urls = [
...(testCase.setup.world?.linkedPullRequests ?? []).map(
({ number }) => pullRequest(repo, { number }).url,
),
...priorCalls.flatMap((call) =>
["link_pull_request", "watch_pull_request"].includes(call.tool) && isRecord(call.input)
? [pullRequest(repo, call.input).url]
: call.tool === "link_pull_requests" &&
isRecord(call.input) &&
Array.isArray(call.input.urls)
? call.input.urls
.filter((url) => PULL_REQUEST_URL.test(valueText(url)))
.map((url) => pullRequest(repo, { url }).url)
: [],
),
];
const taskNumbers = [...valueText(prior).matchAll(/task-(\d+)/g)].map((match) =>
Number(match[1]),
);
const shell = priorCalls
.filter((call) => SHELL_TOOLS.has(call.tool))
.map(({ tool, input, result }) => ({ tool, input, output: valueText(result) }));
return {
testCase,
base,
tools: [...base.nativeTools],
repo: testCase.setup.world?.repo ?? "acme/app",
shell,
shellUsage: emptyUsage(),
linked: new Set([...urls, ...shell.flatMap(createdPullRequests)]),
codeStore: {},
nextTask: Math.max(0, ...taskNumbers) + 1,
threads: new Map(),
secretFiles: [],
secretRequests: 0,
priorResults: new Map(),
};
};
/**
* Builds T3's device tools for a world with devices. Prior device calls whose result is "default"
* run first, in order and without the case's mocks, so their results and sessions are T3's. As in
* the real flow, device_open starts no agent-device session: one starts at a launcher `open` in the
* sample's shell history, prior turns included, or where `world.agentDeviceSessions` declares it.
*/
export const withDevices = Effect.fn("evaluateMcpToolUse.withDevices")(function* (sample: Sample) {
const world = sample.testCase.setup.world;
if (world?.devices === undefined) return;
sample.devices = yield* makeDeviceTools(
{ ...world, devices: world.devices },
sample.base.environment,
);
for (const call of (sample.testCase.setup.priorTurns ?? []).flatMap((t) => t.toolCalls ?? [])) {
if (call.result !== "default" || !isDeviceTool(call.tool)) continue;
sample.priorResults.set(call, (yield* sample.devices.call(call.tool, call.input)).text);
}
});
// Main's history under every case's world.
const MAIN_COMMITS = [
{ sha: "9f2c1ab", subject: "feat(web): show the model name in the thread header" },
{ sha: "4e7d2c0", subject: "fix(server): retry provider start once on ENOENT" },
{ sha: "b81f3d9", subject: "chore: update dependencies" },
];
/**
* The gitStatus Claude Code shows when a session starts, built from the case's world. Commits and
* a branch that the prior turns made (their tool results show it) came later, so they are left out.
*/
const gitStatus = ({ setup }: McpToolUseCase) => {
const prior = valueText(setup.priorTurns ?? []);
const branch = setup.world?.branch ?? "main";
const commits = [...(setup.world?.commits ?? []), ...MAIN_COMMITS].filter(
(commit) => !prior.includes(` ${commit.sha}] `),
);
return [
"As you answer the user's questions, you can use the following context:",
"# gitStatus",
"This is the git status at the start of the conversation. Note that this status is a snapshot in time, and will not update during the conversation.",
"",
`Current branch: ${prior.includes(`Switched to a new branch '${branch}'`) ? "main" : branch}`,
"",
"Main branch (you will usually use this for PRs): main",
"",
"Status:",
setup.world?.status || "(clean)",
"",
"Recent commits:",
...commits.slice(0, 5).map((commit) => `${commit.sha} ${commit.subject}`),
"",
"Claude Code attached this context automatically; it isn't part of the user's message. It describes the user's own account and workspace, so they don't need it reported back.",
].join("\n");
};
const findTool = (base: Base, bare: string) =>
[...base.nativeTools, ...base.deferred].find((tool) => normalizeToolName(tool.name) === bare) ??
(base.codex ? { name: bare } : undefined);
/** Adds deferred schemas to `tools` and returns the ToolSearch result Claude Code sends. */
const loadTools = (tools: Array<ApiTool>, id: string, found: ReadonlyArray<ApiTool>): Block => {
for (const tool of found) {
if (!hasTool(tools, tool.name)) tools.push({ ...tool, defer_loading: true });
}
const content = found.map((tool) => ({ type: "tool_reference", tool_name: tool.name }));
return { type: "tool_result", tool_use_id: id, content };
};
/** Messages for a case: reminders, context, prior turns with real tool blocks, then the prompt. */
export const buildMessages = (sample: Sample) => {
const { testCase, base, tools } = sample;
const messages: Array<ApiMessage> = [];
const push = (role: ApiMessage["role"], ...blocks: ReadonlyArray<Block>) => {
const last = messages.at(-1);
if (last?.role !== role) messages.push({ role, content: blocks });
else messages[messages.length - 1] = { role, content: [...last.content, ...blocks] };
};
const { role, context, delegateRole, priorTurns = [] } = testCase.setup;
const sideChat = testCase.setup.sideChat;
if (testCase.prompt === "" && (role !== "top" || sideChat !== undefined)) {
throw new Error(
`${testCase.id}: Empty-prompt replay requires a top-level thread without side-chat context.`,
);
}
const sideChatContext =
sideChat === undefined
? undefined
: makeSideChatContext({
parentThreadId: ThreadId.make(sideChat.parentThreadId),
digest: sideChat.digest,
});
const native = role === "native-subagent";
const notes = [
native
? "You are a subagent started by the parent agent's Agent tool. Your final message goes to the parent agent, not to the user."
: undefined,
context,
];
const reminders = base.reminders.toSpliced(base.gitStatusAt, 0, reminder(gitStatus(testCase)));
push("user", ...reminders, ...notes.flatMap((note) => (note ? [reminder(note)] : [])));
const toolSearch = findTool(base, "ToolSearch")?.name ?? "ToolSearch";
let nextId = 0;
for (const turn of priorTurns) {
const calls = (turn.toolCalls ?? []).map((call) => {
const tool = findTool(base, call.tool);
if (tool === undefined) throw new Error(`${testCase.id}: unknown prior tool ${call.tool}.`);
if (call.result === "default" && isDeviceTool(call.tool) && !sample.priorResults.has(call)) {
throw new Error(`${testCase.id}: prior ${call.tool} needs T3's result; run withDevices.`);
}
return { ...call, id: `toolu_prior_${nextId++}`, name: tool.name, tool, prior: call };
});
// Prior T3 calls load their schemas first, as in a real deferred session.
const unloaded = [...new Set(calls.map((call) => call.tool))].filter(
(tool) => !hasTool(tools, tool.name),
);
if (unloaded.length > 0 && !base.codex) {
const id = `toolu_prior_${nextId++}`;
const query = `select:${unloaded.map((tool) => tool.name).join(",")}`;
const input = { query, max_results: unloaded.length };
push("assistant", { type: "tool_use", id, name: toolSearch, input });
push("user", loadTools(tools, id, unloaded), text(TOOL_LOADED));
}
// One call per step, as most real turns run them.
for (const { id, name, input, result, tool, prior } of calls) {
push("assistant", { type: "tool_use", id, name, input });
const replayed =
normalizeToolName(tool.name) === "orchestrator_capabilities"
? capabilities(sample, result)
: (sample.priorResults.get(prior) ?? result);
push("user", toolResult(id, replayed));
}
if (turn.text) push(turn.role, text(turn.text));
}
const userText =
testCase.promptContext === undefined
? testCase.prompt
: projectComposerContextForProvider({ text: testCase.prompt, ...testCase.promptContext });
const prompt =
sideChatContext === undefined ? userText : withInlineContext(sideChatContext, userText);
const childPrompt = () => taskPrompt({ task: prompt, role: delegateRole });
if (testCase.prompt !== "")
push("user", text(role === "delegated-child" ? childPrompt() : prompt));
// Claude Code gives a subagent its SubagentStart hooks' context as a reminder after the task.
if (native && base.subagentContext) {
push("user", reminder(`SubagentStart hook additional context: ${base.subagentContext}`));
}
return messages;
};
// Results for T3 tools a case does not mock. The model ids are public in this repo's catalog.
const claudeModel = (id: string) => ({ id, options: [{ id: "effort", values: EFFORTS }] });
const codexModel = (id: string) => ({ id, options: [{ id: "reasoningEffort", values: EFFORTS }] });
// Mirrors the real catalog's main models, so a case's named model is not missing by accident.
const PROVIDERS = [
{
providerInstanceId: "claudeAgent",
models: ["claude-opus-5-5", "claude-sonnet-5-5", "claude-fable-5-1", "claude-haiku-5-5"].map(
claudeModel,
),
},
{
providerInstanceId: "codex",
models: ["gpt-6.1-sol", "gpt-6-astra", "gpt-6-luna"].map(codexModel),
},
{ providerInstanceId: "muse", models: ["muse-wonder-full", "muse-wonder"].map(codexModel) },
{ providerInstanceId: "grok", models: ["grok-4.7"].map(codexModel) },
];
const uuid = (n: number) => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`;
const DEFAULT_THREAD = {
title: "Fix the flaky reconnect test",
status: "idle",
items: [
{
position: 1,
createdBy: "user",
text: "The reconnect test in src/session.test.ts fails about 1 in 10 runs. Fix it.",
},
{
position: 2,
createdBy: "assistant",
text: "Fixed. The test now waits for the socket close event before it reconnects. 20 runs in a row pass.",
},
],
};
const HTML_RENDERED =
"Shown to the reader above your reply. Don't mention or describe the page; reply with only what it doesn't already say.";
const PULL_REQUEST_URL = /\/pull\/\d+\/?$/;
/** Pull request identity from a `url` or `repository` plus `number` input, like T3's results. */
const pullRequest = (repo: string, input: Readonly<Record<string, unknown>>) => {
const url = typeof input.url === "string" ? input.url : "";
const repository =
/github\.com\/([^/]+\/[^/]+)\/pull/.exec(url)?.[1] ?? valueText(input.repository ?? repo);
const number = Number(/(\d+)\/?$/.exec(url)?.[1] ?? input.number);
return {
host: "github.com",
repository,
number,
url: url || `https://github.com/${repository}/pull/${number}`,
};
};
/** A linked PR as `list_thread_pull_requests` shows it. Links made in the sample are open. */
const linkedEntry = (sample: Sample, url: string) => {
const pr = pullRequest(sample.repo, { url });
const seeded = sample.testCase.setup.world?.linkedPullRequests?.find(
({ number }) => number === pr.number && pr.repository === sample.repo,
);
return { ...pr, state: seeded?.state ?? "open", title: seeded?.title ?? null };
};
/** A believable T3 result for a tool the case does not mock. */
export const t3Default = (
sample: Sample,
name: string,
input: Readonly<Record<string, unknown>>,
) => {
const pr = () => pullRequest(sample.repo, input);
const newThread = (request: unknown, brief: unknown) => {
const n = sample.nextTask++;
const title =
isRecord(request) && typeof request.title === "string" ? request.title : "New thread";
const text = typeof brief === "string" ? brief.trim() : "";
sample.threads.set(uuid(n), { title, brief: text });
return {
threadId: uuid(n),
title,
...(text ? { runId: `run-${n}`, status: "starting" } : { status: "idle" }),
};
};
switch (name) {
case "t3_thread_launch":
return newThread(input, input.message);
case "create_threads":
return {
threads: (Array.isArray(input.threads) ? input.threads : []).map((request) => {
const target = isRecord(request) && isRecord(request.target) ? request.target : {};
return {
...newThread(request, isRecord(request) ? request.prompt : undefined),
createdBy: "agent",
creationSource: "mcp",
providerInstanceId:
target.providerInstanceId ?? (sample.base.codex ? "codex" : "claudeAgent"),
model: target.model ?? sample.base.model,
};
}),
};
case "orchestrator_capabilities":
return {
inheritedProviderInstanceId: "claudeAgent",
inheritedModel: sample.base.model,
providers: PROVIDERS,
};
case "delegate_task": {
const n = sample.nextTask++;
const target = isRecord(input.target) ? input.target : {};
return {
taskId: `task-${n}`,
childThreadId: uuid(n),
childRunId: `run-${n}`,
status: "running",
workState: "working",
hasPendingChildRuns: false,
providerInstanceId:
target.providerInstanceId ?? (sample.base.codex ? "codex" : "claudeAgent"),
model: target.model ?? sample.base.model,
summary: null,
hasPendingQuestion: false,
pendingApproval: null,
};
}
case "task_status":
return {
taskId: input.taskId,
status: "running",
workState: "working",
hasPendingChildRuns: false,
summary: null,
};
case "task_cancel":
return { taskId: input.taskId, status: "cancel_requested" };
case "link_pull_request": {
const alreadyLinked = sample.linked.has(pr().url);
sample.linked.add(pr().url);
return { ...pr(), alreadyLinked };
}
case "link_pull_requests": {
const urls = Array.isArray(input.urls) ? input.urls.map(valueText) : [];
const invalidUrls = urls.filter((url) => !PULL_REQUEST_URL.test(url));
const targets = [...new Set(urls.filter((url) => PULL_REQUEST_URL.test(url)))].map((url) =>
pullRequest(sample.repo, { url }),
);
const fresh = targets.filter(({ url }) => !sample.linked.has(url));
for (const { url } of targets) sample.linked.add(url);
const byRepository = (prs: ReadonlyArray<ReturnType<typeof pullRequest>>) =>
[...Map.groupBy(prs, (pr) => pr.repository).values()].map((group) => ({
host: "github.com",
repository: group[0]!.repository,
numbers: group.map((pr) => pr.number),
}));
return {
linked: byRepository(fresh),
alreadyLinked: byRepository(targets.filter((pr) => !fresh.includes(pr))),
linkedCount: targets.length,
...(invalidUrls.length > 0 ? { invalidUrls } : {}),
};
}
case "request_secret": {
// T3 names a ref and a file with 32 hex characters, new for each request, in its secrets dir.
const id = NodeCrypto.createHash("sha256")
.update(`${sample.testCase.id}:${sample.secretRequests++}`)
.digest("hex")
.slice(0, 32);
const home =
/^\/(?:home|Users)\/[^/]+/.exec(sample.base.environment.cwd)?.[0] ?? "/home/user";
return input.delivery === "file" || pastedSecret(name, input)
? { status: "saved", file: `${home}/.t3/userdata/secrets/files/${id}` }
: { status: "saved", secretRef: `secret-ref:${id}` };
}
case "unlink_pull_request":
return { ...pr(), wasLinked: sample.linked.delete(pr().url) };
case "unlink_pull_requests": {
const states = Array.isArray(input.states) ? input.states : [];
const unlinked = [...sample.linked]
.map((url) => linkedEntry(sample, url))
.filter((entry) => states.includes(entry.state));
for (const { url } of unlinked) sample.linked.delete(url);
const groups = Map.groupBy(unlinked, (entry) => `${entry.repository}/${entry.state}`);
return {
unlinked: [...groups.values()].map((entries) => ({
host: "github.com",
repository: entries[0]!.repository,
state: entries[0]!.state,
numbers: entries.map((entry) => entry.number),
})),
unlinkedCount: unlinked.length,
remaining: sample.linked.size,
};
}
case "watch_pull_request":
sample.linked.add(pr().url);
return { ...pr(), watching: true, wasWatching: false };
case "unwatch_pull_request":
return { ...pr(), watching: false, wasWatching: true };
case "list_thread_pull_requests":
return {
pullRequests: [...sample.linked].map((url) => ({
...linkedEntry(sample, url),
source: "agent",
watching: false,
})),
chains: [],
};
case "t3_thread_read": {
const threadId = valueText(input.threadId);
const launched = sample.threads.get(threadId);
if (launched !== undefined) {
const items = launched.brief
? [{ position: 1, createdBy: "agent", creationSource: "mcp", text: launched.brief }]
: [];
const thread = {
threadId,
title: launched.title,
createdBy: "agent",
creationSource: "mcp",
status: launched.brief ? "running" : "idle",
};
return { thread, items, nextPosition: items.length + 1, hasMore: false };
}
const { title, status, items } = DEFAULT_THREAD;
return { thread: { threadId, title, status }, items, nextPosition: 3, hasMore: false };
}
case "html_preview":
return { width: 728, contentHeight: 560, capturedHeight: 560, consoleMessages: [] };
case "html_render":
return {
htmlRender: { attachmentId: "att-1", title: input.title ?? "Visual", height: 560 },
message: HTML_RENDERED,
};
case "preview_status":
return {
available: false,
visible: false,
tabId: null,
url: null,
title: null,
loading: false,
};
default:
return { ok: true };
}
};
/** The machine the fake threads live on. */
const EVAL_ENVIRONMENT_ID = EnvironmentId.make("eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee");
// The note T3 adds when a secret card ends unsaved, on refs that have one (looked up by name so
// the runner still builds on refs without it).
const unsavedSecretNote = (Contracts as Record<string, unknown>).SECRET_REQUEST_UNSAVED_NOTE as
| Readonly<Record<string, string>>
| undefined;
/** Fields T3 adds to real thread and task results, applied to mocks and defaults alike. */
const withProductFields = (name: string, value: unknown, child: boolean): unknown => {
if (!isRecord(value)) return value;
const linked = (thread: unknown, idKey = "threadId") => {
if (!isRecord(thread) || thread.link !== undefined) return thread;
const id = thread[idKey] ?? thread.id;
return typeof id === "string"
? {
...thread,
link: formatThreadLink({
environmentId: EVAL_ENVIRONMENT_ID,
threadId: ThreadId.make(id),
}),
}
: thread;
};
const listed = (key: string) =>
Array.isArray(value[key]) ? { ...value, [key]: value[key].map((item) => linked(item)) } : value;
switch (name) {
case "request_secret": {
const note = typeof value.status === "string" ? unsavedSecretNote?.[value.status] : undefined;
return note === undefined || value.note !== undefined ? value : { ...value, note };
}
case "delegate_task":
case "task_status": {
const task = linked(value, "childThreadId") as Record<string, unknown>;
const running = task.status === "running" && task.next === undefined;
return running ? { ...task, next: ORCHESTRATOR_MCP_RUNNING_TASK_NEXT } : task;
}
case "t3_thread_read":
return { ...value, thread: linked(value.thread) };
case "t3_thread_launch":
case "t3_thread_send":
case "run_scheduled_task_now":
return linked(value);
case "t3_thread_list":
case "create_threads":
return listed("threads");
case "t3_thread_search":
return listed("matches");
case "link_pull_request":
case "link_pull_requests":
return child ? { ...value, next: CHILD_PULL_REQUEST_REPORT_NEXT } : value;
default:
return value;
}
};
/** Cases mock a Claude caller, so the inherited fields follow the model under test. */
const capabilities = ({ base }: Sample, value: unknown) =>
isRecord(value)
? {
...value,
inheritedModel: base.model,
...(base.codex
? {
inheritedProviderInstanceId: "codex",
inheritedOptions: [{ id: "reasoningEffort", value: base.codex.reasoning.effort }],
}
: {}),
}
: value;
/** ToolSearch: `select:a,b` loads those tools. Other queries load tools whose name has a word. */
const searchTools = (base: Base, query: string, max: number) => {
if (query.startsWith("select:")) {
const wanted = query.slice("select:".length).split(",");
return base.deferred.filter((tool) =>
wanted.some((name) => [tool.name, normalizeToolName(tool.name)].includes(name.trim())),
);
}
const words = query
.toLowerCase()
.split(/[\s+]+/)
.filter(Boolean);
const hits = (tool: ApiTool) => words.filter((word) => tool.name.toLowerCase().includes(word));
const found = base.deferred.filter((tool) => hits(tool).length > 0);
return found.toSorted((a, b) => hits(b).length - hits(a).length).slice(0, max);
};
const SUBAGENT_WATCH_ERROR =
"This thread is a subagent, so it cannot watch pull requests. Its parent thread owns the pull request: finish your task and report back instead.";
/** Result for a native tool the case does not mock, other than shell, file, and web tools. */
const nativeResult = (name: string, input: Readonly<Record<string, unknown>>) => {
const skill = valueText(input.skill);
const results: Readonly<Record<string, string>> = {
Agent: "The agent finished. Summary: the task is done and nothing else needs attention.",
Skill: `Launching skill: ${skill}\n\nBase directory for this skill: ~/.claude/skills/${skill}\n\nThis skill adds no steps for this task. Do it with your usual tools.`,
WebSearch: "No results found.",
};
return results[name] ?? "OK";
};
const SHELL_PROMPT = `You play the computer a coding agent works on. You get a description of the machine, the agent's tool calls so far with their outputs, and one new call. Reply in JSON. In "facts", note briefly what the output must agree with. In "output", put only the call's output, exactly as the tool prints it: no notes, headings, code fences, or corrections. The agent sees "output" unchanged.
- Bash: what the command prints in a real terminal (git, gh, ls, cat, grep, package managers). If it fails, start with "Exit code N", then the error. If it prints nothing, reply "(Bash completed with no output)".
- Read: the file with numbered lines, like \`cat -n\`. Write: "File created successfully at: <path>". Edit: "The file <path> has been updated successfully." Edit may instead supply a Codex apply_patch patch; apply its additions, deletions and hunks. A missing file, or an Edit whose old_string is not in the file: "<tool_use_error>reason</tool_use_error>".
- WebFetch: the page's text.
Stay consistent with the machine and with earlier outputs: edits, commits, pushes, and new PRs show in later output. Where the description is silent, invent plausible small details that fit it (file contents, SHAs, test names) rather than printing nothing. Commands succeed unless the description says otherwise, including git push, gh pr create, installs, builds, and tests. A new PR gets the next number after the highest one in the description or the history, or 124 if there is none.`;
const ShellReply = Schema.Struct({ facts: Schema.String, output: Schema.String });
const SHELL_REPLY_SCHEMA = {
...Schema.toJsonSchemaDocument(ShellReply).schema,
additionalProperties: false,
};
const decodeShellReply = Schema.decodeUnknownOption(Schema.fromJsonString(ShellReply));
const callText = ({ tool, input }: Omit<ShellEntry, "output">) =>
`<call>${tool} ${valueText(input)}</call>`;
/**
* Absolute paths the case's context or files name exist, wherever the template's working
* directory is. Without this the simulator denies a checkout the agent was told to work in.
*/
export const namedPathsLine = ({ setup }: McpToolUseCase) => {
const absolute = /(?:^|[\s(`'"])((?:\/home|\/Users|\/tmp)\/[^\s`'",;:)]+)/g;
const fromContext = [...(setup.context ?? "").matchAll(absolute)].flatMap(([, path]) =>
path ? [path] : [],
);
const fromFiles = Object.keys(isRecord(setup.world?.files) ? setup.world.files : {}).filter(
(path) => path.startsWith("/"),
);
const paths = [
...new Set([...fromContext, ...fromFiles].map((path) => path.replace(/[.]+$/, ""))),
];
return paths.length > 0
? [
`These paths exist on this machine, with the contents the world gives: ${paths.join(", ")}. The working directory above is only where the shell starts; the agent may cd into them.`,
]
: [];
};
/** The simulator's request: the machine, this sample's earlier calls, then the new call. */
export const shellRequest = (sample: Sample, call: Omit<ShellEntry, "output">, model: string) => {
const { testCase, base, repo, shell, devices } = sample;
const { world, context, shellMocks = [] } = testCase.setup;
const { cwd, platform, date } = base.environment;
const command = call.tool === "Bash" && isRecord(call.input) ? call.input.command : undefined;
const machine = [
`Working directory: ${cwd} on ${platform}, a git checkout of github.com/${repo}.${date ? ` Today is ${date}.` : ""}`,
gitStatus(testCase),
`World: ${valueText(world ?? {})}`,
...(shellMocks.length > 0
? [`Commands that match these regexes print exactly this output: ${valueText(shellMocks)}`]
: []),
// Which device each launcher and session drives, and which of this call's opens are refused.
...(devices?.describe(shell, typeof command === "string" ? command : undefined) ?? []),
...(world?.deviceInvalidCapture
? [
"T3's device_screenshot gets each capture with adb screencap's multi-display warning ahead of the PNG, so it reports 0×0. agent-device's own screenshot finds the PNG and is unaffected.",
]
: []),
...(sample.secretFiles.length > 0
? [
`T3's request_secret saved these files in this sample: ${sample.secretFiles.join(", ")}. Each exists, is readable only by the user, and holds exactly the value the user gave, typed into T3's private card or pasted in chat and saved by the agent, the right one for the task, with no trailing newline. A program that reads it, as in "$(cat <file>)", < <file> or a --*-file flag, gets that value. What appears in the output is whatever that program prints: one that prints or logs its input shows the value. No other file in that directory exists.`,
]
: []),
...(context ? [`The agent was told: ${context}`] : []),
`The agent's task: ${testCase.prompt}`,
...namedPathsLine(testCase),
];
const history = shell.map((entry) => `${callText(entry)}\n<output>\n${entry.output}\n</output>`);
return {
model,
max_tokens: 8_000,
// The "facts" field keeps the simulator's reasoning out of the output the agent sees.
output_config: { effort: "low", format: { type: "json_schema", schema: SHELL_REPLY_SCHEMA } },
// A case with devices also gets agent-device's own CLI, so T3's launchers answer as it does.
system: devices ? `${SHELL_PROMPT}\n\n${AGENT_DEVICE_SHELL}` : SHELL_PROMPT,
messages: [
{
role: "user",