/** Pure grading for the mobile behavior eval: frame and scroll math, checks, and the summary. */
import type { FeedSample } from "./device.ts";
export type Metrics = Readonly<Record<string, number | boolean | string | null>>;
export interface Check {
readonly metric: string;
readonly max?: number;
readonly min?: number;
readonly equals?: number | boolean | string | null;
}
const percentile = (sorted: ReadonlyArray<number>, p: number) =>
sorted.length === 0 ? 0 : sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))]!;
export function median(values: ReadonlyArray<number>): number {
const sorted = [...values].sort((a, b) => a - b);
const middle = Math.floor(sorted.length / 2);
return sorted.length % 2 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2;
}
const round = (value: number, digits = 1) => Number(value.toFixed(digits));
/**
* Frame times from rAF timestamps, graded against the display's budget: `refreshMs` when given,
* else 120 Hz when the fastest frames come under 12 ms and 60 Hz otherwise. A steady low frame
* rate still counts as dropped frames. `dropped` counts the frames that should have run inside
* each long gap.
*/
export function frameStats(
timestamps: ReadonlyArray<number>,
prefix: string,
refreshMs?: number,
): Metrics {
const gaps = timestamps.slice(1).map((time, index) => time - timestamps[index]!);
if (gaps.length < 10) return { [`${prefix}Frames`]: gaps.length };
const sorted = [...gaps].sort((a, b) => a - b);
const refresh = refreshMs ?? (percentile(sorted, 0.05) < 12 ? 1000 / 120 : 1000 / 60);
const dropped = gaps.reduce(
(total, gap) => total + Math.max(0, Math.round(gap / refresh) - 1),
0,
);
return {
[`${prefix}Frames`]: gaps.length,
[`${prefix}RefreshHz`]: Math.round(1000 / refresh),
[`${prefix}P50Ms`]: round(percentile(sorted, 0.5)),
[`${prefix}P95Ms`]: round(percentile(sorted, 0.95)),
[`${prefix}P99Ms`]: round(percentile(sorted, 0.99)),
[`${prefix}MaxMs`]: round(sorted.at(-1)!),
[`${prefix}DroppedPct`]: round((100 * dropped) / (gaps.length + dropped)),
};
}
/** Fastest real fling step: about 8000 pt/s at 60 Hz. A bigger one-frame move is a jump. */
export const TELEPORT_PX = 400;
/**
* How visible rows moved between JS frames. A row's screen y is its list position minus the
* scroll offset, so a list that shifts its offset to hold content still reads as no motion.
* `direction` is the way rows should move: +1 while dragging content down (scrolling up).
*/
export function feedMotion(samples: ReadonlyArray<FeedSample>, direction: 1 | -1 | 0): Metrics {
let backward = 0;
let largest = 0;
let teleports = 0;
let moved = 0;
for (let index = 1; index < samples.length; index++) {
const before = samples[index - 1]!;
const after = samples[index]!;
const shared = Object.keys(before.rows ?? {}).filter((key) => after.rows?.[key] !== undefined);
let step = 0;
for (const key of shared) {
const rowStep = after.rows![key]! - before.rows![key]!;
if (Math.abs(rowStep) > Math.abs(step)) step = rowStep;
}
// A jump can replace every visible row; with the same content, the offset change is the move.
if (shared.length === 0 && before.content === after.content)
step = (before.scroll ?? 0) - (after.scroll ?? 0);
moved += Math.abs(step);
largest = Math.max(largest, Math.abs(step));
if (Math.abs(step) > TELEPORT_PX) teleports++;
if (direction !== 0) backward = Math.max(backward, -step * direction);
}
const fromEnd = samples.flatMap((sample) =>
sample.fromEnd === undefined ? [] : [sample.fromEnd],
);
return {
samples: samples.length,
movedPx: Math.round(moved),
backwardPx: Math.round(backward),
largestStepPx: Math.round(largest),
teleports,
maxFromEndPx: fromEnd.length ? Math.max(...fromEnd) : null,
finalFromEndPx: fromEnd.at(-1) ?? null,
};
}
export function checkFails(check: Check, metrics: Metrics): string | null {
const value = metrics[check.metric];
if (value === undefined) return `${check.metric} missing`;
if (check.equals !== undefined)
return value === check.equals ? null : `${check.metric}=${value}, want ${check.equals}`;
if (typeof value !== "number") return `${check.metric}=${value}, want a number`;
if (check.max !== undefined && value > check.max)
return `${check.metric}=${value} > ${check.max}`;
if (check.min !== undefined && value < check.min)
return `${check.metric}=${value} < ${check.min}`;
return null;
}
export interface SampleRow {
readonly id: string;
readonly device: string;
readonly sample: number;
readonly pass: boolean | null;
readonly failures: ReadonlyArray<string>;
readonly metrics: Metrics;
/** Why the sample has no grade: the scenario failed before it could measure. */
readonly error?: string;
}
export interface CaseResult {
readonly id: string;
readonly device: string;
readonly samples: number;
readonly passed: number;
readonly errors: number;
/**
* `null` when too few samples were graded: errors outnumber graded samples, or none ran.
* Otherwise a case fails when half or more of its graded samples fail.
*/
readonly pass: boolean | null;
readonly key: number | boolean | string | null;
}
export function caseResult(
id: string,
device: string,
key: string,
rows: ReadonlyArray<SampleRow>,
): CaseResult {
const graded = rows.filter((row) => row.pass !== null);
const errors = rows.filter((row) => row.error !== undefined).length;
const passed = graded.filter((row) => row.pass).length;
const values = graded.map((row) => row.metrics[key]);
const numbers = values.filter((value): value is number => typeof value === "number");
return {
id,
device,
samples: graded.length,
passed,
errors,
pass: graded.length === 0 || errors > graded.length ? null : passed > graded.length / 2,
key:
numbers.length === values.length && numbers.length > 0
? round(median(numbers))
: (values[Math.floor(values.length / 2)] ?? null),
};
}
const mark = (result: CaseResult) =>
`${result.pass === null ? "incomplete" : result.pass ? "pass" : "FAIL"} ${result.passed}/${result.samples + result.errors}`;
/** Markdown scoreboard. With a baseline, each row shows the earlier result. */
export function summarize(input: {
readonly cases: ReadonlyArray<{
readonly id: string;
readonly kind: string;
readonly source: string;
readonly key: string;
}>;
readonly results: ReadonlyArray<CaseResult>;
readonly baseline?: ReadonlyArray<CaseResult>;
readonly header: string;
}): string {
const devices = [...new Set(input.results.map((result) => result.device))];
const cell = (result: CaseResult | undefined) =>
result ? `${mark(result)} (${result.key ?? "-"})` : "-";
const lines = [
input.header,
"",
`| Case | Kind | Source | Key metric | ${devices.join(" | ")} |`,
`| --- | --- | --- | --- | ${devices.map(() => "---").join(" | ")} |`,
];
for (const testCase of input.cases) {
const cells = devices.map((device) => {
const now = input.results.find((r) => r.id === testCase.id && r.device === device);
const before = input.baseline?.find((r) => r.id === testCase.id && r.device === device);
return before ? `${cell(now)} was ${cell(before)}` : cell(now);
});
if (cells.every((value) => value === "-")) continue;
lines.push(
`| \`${testCase.id}\` | ${testCase.kind} | ${testCase.source} | ${testCase.key} | ${cells.join(" | ")} |`,
);
}
const ran = input.results.filter((result) => result.pass !== null);
const incomplete = input.results.length - ran.length;
lines.push(
"",
`${ran.filter((r) => r.pass).length} of ${ran.length} case runs pass${incomplete ? `, ${incomplete} incomplete` : ""}.`,
);
return `${lines.join("\n")}\n`;
}
export function resultsFor(
rows: ReadonlyArray<SampleRow>,
cases: ReadonlyArray<{ readonly id: string; readonly key: string }>,
): ReadonlyArray<CaseResult> {
const keys = [...new Set(rows.map((row) => `${row.id}\u0000${row.device}`))];
return keys.map((key) => {
const [id, device] = key.split("\u0000") as [string, string];
return caseResult(
id,
device,
cases.find((entry) => entry.id === id)?.key ?? "",
rows.filter((row) => row.id === id && row.device === device),
);
});
}
/**
* Results that should stop a release: a guard or control that failed or could not be graded.
* `bad` cases reproduce known bugs and fail until those are fixed.
*/
export function blockingResults(
results: ReadonlyArray<CaseResult>,
cases: ReadonlyArray<{ readonly id: string; readonly kind: string }>,
): ReadonlyArray<CaseResult> {
return results.filter(
(result) =>
result.pass !== true && cases.find((entry) => entry.id === result.id)?.kind !== "bad",
);
}