/**
* Cases for the mobile behavior eval. `bad` fails on main today and must pass once fixed;
* `guard` passes on main and must keep passing (its issue is fixed, or does not show on iOS);
* `control` is ordinary use that must keep passing. Each names the scenario that produces its metrics and the checks on them.
*/
import type { Check } from "./grade.ts";
import type { Pose, ScenarioName } from "./scenarios.ts";
export interface MobileBehaviorCase {
readonly id: string;
readonly area: "scroll" | "jank" | "composer" | "status" | "wrong-thread";
readonly kind: "bad" | "guard" | "control";
/** The issue or PR it came from. */
readonly source: string;
/** The real bad experience, or the good one a guard protects. */
readonly experience: string;
readonly scenario: ScenarioName;
/** The metric the scoreboard shows. */
readonly key: string;
readonly checks: ReadonlyArray<Check>;
/** Omit to run on every pose. */
readonly poses?: ReadonlyArray<Pose>;
}
/**
* Frame budget checks for jank cases: at most 5% of UI frames and 10% of JS frames dropped.
* Swipe-driven cases check JS frames only: the injected swipes cost the simulator's UI thread.
*/
const smooth: ReadonlyArray<Check> = [
{ metric: "uiDroppedPct", max: 5 },
{ metric: "jsDroppedPct", max: 10 },
];
export const CASES: ReadonlyArray<MobileBehaviorCase> = [
{
id: "scroll-up-never-moves-backward",
area: "scroll",
kind: "guard",
source: "#22350",
experience: "Flicking up a long thread, content jerks back down toward the latest message.",
scenario: "scrollUpLongThread",
key: "backwardPx",
checks: [
{ metric: "movedPx", min: 300 },
{ metric: "backwardPx", max: 24 },
],
},
{
id: "scroll-up-never-teleports",
area: "scroll",
kind: "guard",
source: "#12072",
experience: "Dragging up a little after scroll-to-end, the viewport jumps far away.",
scenario: "scrollUpLongThread",
key: "largestStepPx",
checks: [
{ metric: "movedPx", min: 300 },
{ metric: "teleports", equals: 0 },
],
},
{
id: "streaming-stays-at-latest",
area: "scroll",
kind: "bad",
source: "#15469",
experience:
"While a reply streams at the bottom, the thread jumps far above the latest message.",
scenario: "streamInView",
key: "settledFromEndPx",
checks: [
{ metric: "settledFromEndPx", max: 64 },
{ metric: "followingAtEnd", equals: true },
],
},
{
id: "send-in-long-thread-stays-at-end",
area: "scroll",
kind: "guard",
source: "#13706",
experience:
"Sending in a long thread, the timeline jumps far up instead of showing the new message.",
scenario: "sendInLongThread",
key: "maxFromEndPx",
checks: [
{ metric: "maxFromEndPx", max: 1500 },
{ metric: "finalFromEndPx", max: 24 },
],
},
{
id: "background-finish-keeps-scroll",
area: "scroll",
kind: "guard",
source: "#15165",
experience: "Another thread finishing resets the open thread's scroll position.",
scenario: "backgroundFinish",
key: "largestStepPx",
checks: [{ metric: "largestStepPx", max: 4 }],
},
{
id: "jank-streaming-long-reply",
area: "jank",
kind: "bad",
source: "#15641, #14010, #19120",
experience: "A long reply streaming into a long thread freezes or stutters the app.",
scenario: "streamInView",
key: "jsDroppedPct",
checks: smooth,
},
{
id: "jank-scrolling-settled-thread",
area: "jank",
kind: "control",
source: "#13925 (Android)",
experience: "Scrolling a long settled thread drops frames.",
scenario: "scrollUpLongThread",
key: "jsP99Ms",
checks: [
{ metric: "movedPx", min: 300 },
{ metric: "jsDroppedPct", max: 10 },
],
},
{
id: "jank-scrolling-subagent-thread",
area: "jank",
kind: "control",
source: "#18376 (Android)",
experience: "Scrolling a subagent's thread is far less smooth than its parent.",
scenario: "openSubagentThread",
key: "jsP99Ms",
checks: [
{ metric: "movedPx", min: 300 },
{ metric: "jsDroppedPct", max: 10 },
],
},
{
id: "draft-survives-thread-switch",
area: "composer",
kind: "guard",
source: "#17422",
experience: "A typed draft is gone after switching threads and coming back.",
scenario: "draftThreadSwitch",
key: "draftKept",
checks: [
{ metric: "draftKept", equals: true },
{ metric: "editorShowsDraft", equals: true },
],
},
{
id: "draft-never-leaks-to-other-thread",
area: "composer",
kind: "guard",
source: "#7390",
experience: "Text typed in a streaming thread shows up in another thread's composer.",
scenario: "draftThreadSwitch",
key: "leakedIntoOtherEditor",
checks: [
{ metric: "leakedIntoOtherDraft", equals: false },
{ metric: "leakedIntoOtherEditor", equals: false },
],
},
{
id: "draft-survives-background",
area: "composer",
kind: "guard",
source: "#17422, #7390",
experience: "A typed draft is gone after the app goes to the background and back.",
scenario: "draftBackground",
key: "draftKept",
checks: [
{ metric: "draftKept", equals: true },
{ metric: "editorShowsDraft", equals: true },
],
},
{
id: "composer-clears-after-send",
area: "composer",
kind: "guard",
source: "#20982",
experience: "After sending, the message text stays in the composer.",
scenario: "sendInLongThread",
key: "editorCleared",
checks: [
{ metric: "draftCleared", equals: true },
{ metric: "editorCleared", equals: true },
],
},
{
id: "composer-tap-focuses-text",
area: "composer",
kind: "guard",
source: "#24303",
experience: "Tapping the composer's text area opens the model picker instead of the keyboard.",
scenario: "tapComposer",
key: "editorFocused",
checks: [
{ metric: "stayedOnThread", equals: true },
{ metric: "editorFocused", equals: true },
],
},
{
id: "composer-keeps-focus-after-command-pick",
area: "composer",
kind: "guard",
source: "#10956",
experience: "Picking a skill or command from the / menu closes the keyboard.",
scenario: "skillPick",
key: "editorFocused",
checks: [{ metric: "editorFocused", equals: true }],
},
{
id: "failed-send-is-never-silent",
area: "composer",
kind: "guard",
source: "#15801",
experience: "A send the server will reject looks sent, with no error.",
scenario: "failedSend",
// The cell shows which path ran: Send blocked, or a rejected send with an error.
key: "sendBlocked",
checks: [{ metric: "silentFailure", equals: false }],
},
{
id: "working-shelf-clears-when-done",
area: "status",
kind: "bad",
source: "#24901",
experience: "A finished thread stays in the Working shelf.",
scenario: "workingShelfFinish",
key: "shelfCountAfter",
checks: [
{ metric: "shelfShownWhileWorking", equals: true },
{ metric: "shelfCountAfter", equals: 0 },
{ metric: "statusAfter", equals: "ready" },
],
},
{
id: "pending-question-shows-input",
area: "status",
kind: "guard",
source: "#24901",
experience: "A thread waiting on a question shows as Working instead of needing input.",
scenario: "pendingQuestion",
key: "status",
checks: [
{ metric: "status", equals: "input" },
{ metric: "shelfCount", equals: 0 },
],
},
{
id: "running-subagents-not-idle",
area: "status",
kind: "guard",
source: "#16464, #4962",
experience: "A thread with subagents still running looks idle in the list and the thread.",
scenario: "subagentsRunning",
key: "status",
checks: [
{ metric: "status", equals: "working" },
{ metric: "threadShowsStatus", equals: true },
],
},
{
id: "long-press-pin-pins-that-row",
area: "wrong-thread",
kind: "guard",
source: "#20428",
experience: "Pin from a row's long-press menu pins a different thread.",
scenario: "pinFromLongPress",
key: "targetPinned",
checks: [
{ metric: "targetPinned", equals: true },
{ metric: "otherPinned", equals: 0 },
],
},
{
id: "link-opens-new-thread",
area: "wrong-thread",
kind: "guard",
source: "#14689 (web)",
experience: "A link to a just-created thread redirects to another thread.",
scenario: "linkToNewThread",
key: "opensTarget",
checks: [{ metric: "opensTarget", equals: true }],
},
{
id: "link-opens-archived-thread",
area: "wrong-thread",
kind: "guard",
source: "#16443",
experience: "A link to an archived thread sends you Home.",
scenario: "linkToArchivedThread",
key: "opensTarget",
checks: [{ metric: "opensTarget", equals: true }],
},
{
id: "folded-duo-back-opens-right-thread",
area: "wrong-thread",
kind: "guard",
source: "#20633",
experience: "On a folded iPhone Duo, after going back, the next thread opens off screen.",
scenario: "duoBackThenOpen",
key: "titleOnScreen",
poses: ["duo-closed"],
checks: [
{ metric: "opensTarget", equals: true },
{ metric: "titleOnScreen", equals: true },
],
},
{
id: "control-open-thread-from-list",
area: "wrong-thread",
kind: "control",
source: "control",
experience: "Tapping a row opens that thread.",
scenario: "openFromList",
key: "opensTarget",
checks: [{ metric: "opensTarget", equals: true }],
},
{
id: "control-send-reaches-server",
area: "composer",
kind: "control",
source: "control",
experience: "A message sent from the composer reaches the server once.",
scenario: "sendInLongThread",
key: "serverMessages",
checks: [{ metric: "serverMessages", equals: 1 }],
},
{
id: "control-streaming-reply-renders",
area: "scroll",
kind: "control",
source: "control",
experience: "A streaming reply grows in the open thread.",
scenario: "streamInView",
key: "contentGrewPx",
checks: [{ metric: "contentGrewPx", min: 500 }],
},
];