const fs = require("node:fs");
const path = require("node:path");
const ARTIFACT_NAME = "thread-transfer-results";
const RESULT_FILE = "thread-transfer-result.json";
const COMMENT_MARKER = "<!-- t3-thread-transfer-report -->";
const PROVIDERS = ["codex", "claudeAgent"];
const OBSERVED_KEYS = [
"totalWireBytes",
"threadSnapshotWireBytes",
"threadSnapshotDecodedBytes",
"measuredTurnWebSocketWireBytes",
"measuredTurnWebSocketDecodedBytes",
"measuredTurnWebSocketMessages",
];
const CEILING_KEYS = [
"totalWireBytes",
"threadSnapshotWireBytes",
"measuredTurnWebSocketWireBytes",
"measuredTurnWebSocketDecodedBytes",
"measuredTurnWebSocketMessages",
];
const SCENARIO_KEYS = [
"id",
"historyTurns",
"historyCommandToolsPerTurn",
"historyMcpResultBytes",
"measuredCommandTools",
"measuredMcpResultBytes",
];
function resultShaMarker(sha) {
return `<!-- t3-thread-transfer-result-sha:${sha} -->`;
}
function assertObject(value, label) {
if (value === null || typeof value !== "object" || Array.isArray(value)) {
throw new Error(`${label} must be an object`);
}
}
function assertExactKeys(value, expected, label) {
assertObject(value, label);
const actual = Object.keys(value).sort();
const wanted = [...expected].sort();
if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index])) {
throw new Error(`${label} has unexpected fields`);
}
}
function assertMetric(value, label) {
if (!Number.isSafeInteger(value) || value < 0 || value > 1_000_000_000) {
throw new Error(`${label} must be a non-negative safe integer below 1,000,000,000`);
}
}
function validateResult(value) {
assertExactKeys(value, ["schemaVersion", "scenario", "providers"], "result");
if (value.schemaVersion !== 1) {
throw new Error("result.schemaVersion must be 1");
}
assertExactKeys(value.scenario, SCENARIO_KEYS, "result.scenario");
if (!["thread-transfer-v1", "thread-transfer-v2"].includes(value.scenario.id)) {
throw new Error("result.scenario.id is not supported");
}
for (const key of SCENARIO_KEYS.slice(1)) {
assertMetric(value.scenario[key], `result.scenario.${key}`);
}
assertExactKeys(value.providers, PROVIDERS, "result.providers");
for (const provider of PROVIDERS) {
const entry = value.providers[provider];
assertExactKeys(entry, ["observed", "ceiling"], `result.providers.${provider}`);
assertExactKeys(entry.observed, OBSERVED_KEYS, `result.providers.${provider}.observed`);
assertExactKeys(entry.ceiling, CEILING_KEYS, `result.providers.${provider}.ceiling`);
for (const key of OBSERVED_KEYS) {
assertMetric(entry.observed[key], `result.providers.${provider}.observed.${key}`);
}
for (const key of CEILING_KEYS) {
assertMetric(entry.ceiling[key], `result.providers.${provider}.ceiling.${key}`);
}
}
return value;
}
function readResult(directory) {
if (!directory) return undefined;
const file = path.join(directory, RESULT_FILE);
if (!fs.existsSync(file)) return undefined;
const stat = fs.lstatSync(file);
if (!stat.isFile() || stat.size > 64 * 1_024) {
throw new Error("thread transfer result must be a regular file smaller than 64 KiB");
}
return validateResult(JSON.parse(fs.readFileSync(file, "utf8")));
}
function formatBytes(bytes) {
if (bytes < 1_024) return `${bytes} B`;
if (bytes >= 1_024 * 1_024) return `${(bytes / 1_024 / 1_024).toFixed(2)} MiB`;
return `${(bytes / 1_024).toFixed(1)} KiB`;
}
function formatValue(value, kind) {
return kind === "messages" ? value.toLocaleString("en-US") : formatBytes(value);
}
function formatImpact(current, baseline, kind) {
if (baseline === undefined) return "—";
const delta = current - baseline;
const prefix = delta > 0 ? "+" : delta < 0 ? "−" : "";
const magnitude = formatValue(Math.abs(delta), kind);
const percent =
baseline === 0 ? "" : ` (${prefix}${Math.abs((delta / baseline) * 100).toFixed(1)}%)`;
return `${prefix}${magnitude}${percent}`;
}
function sameScenario(left, right) {
return SCENARIO_KEYS.every((key) => left[key] === right[key]);
}
const METRICS = [
{ key: "totalWireBytes", label: "Total thread wire", kind: "bytes" },
{ key: "threadSnapshotWireBytes", label: "Thread snapshot wire", kind: "bytes" },
{
key: "measuredTurnWebSocketWireBytes",
label: "Live turn WebSocket wire",
kind: "bytes",
},
{
key: "measuredTurnWebSocketDecodedBytes",
label: "Live turn WebSocket decoded",
kind: "bytes",
},
{ key: "measuredTurnWebSocketMessages", label: "Live turn messages", kind: "messages" },
];
function renderComment(input) {
const current = input.current;
const baseline = input.baseline;
const comparable = baseline !== undefined && sameScenario(current.scenario, baseline.scenario);
const rows = [];
const ceilingChanges = [];
let failed = false;
for (const provider of PROVIDERS) {
for (const metric of METRICS) {
const observed = current.providers[provider].observed[metric.key];
const ceiling = current.providers[provider].ceiling[metric.key];
const baselineObserved = comparable
? baseline.providers[provider].observed[metric.key]
: undefined;
const pass = observed <= ceiling;
failed ||= !pass;
rows.push(
`| ${provider === "codex" ? "Codex" : "Claude"} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`,
);
if (comparable && baseline.providers[provider].ceiling[metric.key] !== ceiling) {
ceilingChanges.push(
`- ${provider === "codex" ? "Codex" : "Claude"} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`,
);
}
}
}
const baselineLink = input.baselineRun
? `[\`${input.baselineRun.sha.slice(0, 7)}\`](${input.baselineRun.url})`
: "unavailable";
const currentLink = `[\`${input.currentRun.sha.slice(0, 7)}\`](${input.currentRun.url})`;
const notices = [];
if (!baseline) {
notices.push(
"> ℹ️ No successful `main` baseline artifact is available yet. This run establishes the initial measurement.",
);
} else if (!comparable) {
notices.push(
"> ⚠️ The thread fixture changed, so impact percentages are not directly comparable to the `main` baseline.",
);
} else if (!input.baselineRun.matchesBase) {
notices.push(
"> ℹ️ The exact PR base did not have a successful artifact. Baseline uses the latest successful `main` measurement shown below.",
);
}
if (ceilingChanges.length > 0) {
notices.push(
`> ⚠️ **This PR changes transfer ceilings:**\n>\n${ceilingChanges.map((line) => `> ${line}`).join("\n")}`,
);
}
return [
COMMENT_MARKER,
resultShaMarker(input.currentRun.sha),
"## Thread transfer impact",
"",
failed
? "❌ One or more thread transfer ceilings were exceeded."
: "✅ Thread transfer remains within every enforced ceiling.",
...(notices.length > 0 ? ["", ...notices] : []),
"",
"| Provider | Metric | Main baseline | This PR | Impact | PR ceiling | |",
"| --- | --- | ---: | ---: | ---: | ---: | --- |",
...rows,
"",
`Baseline: ${baselineLink} · PR result: ${currentLink} · Source CI: ${input.currentRun.conclusion}`,
"",
"<details>",
"<summary>Scenario and decoded snapshot size</summary>",
"",
`${current.scenario.historyTurns} historical turns, ${current.scenario.historyCommandToolsPerTurn} command tools per turn, ${formatBytes(current.scenario.historyMcpResultBytes)} retained MCP result per historical turn, and a ${formatBytes(current.scenario.measuredMcpResultBytes)} retained result in the measured turn.`,
"",
...PROVIDERS.map(
(provider) =>
`- ${provider === "codex" ? "Codex" : "Claude"} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`,
),
"",
"</details>",
"",
"_Updated in place by a trusted workflow. PR artifacts are strictly validated and never executed._",
].join("\n");
}
async function artifactsForRun(github, owner, repo, runId) {
return github.paginate(github.rest.actions.listWorkflowRunArtifacts, {
owner,
repo,
run_id: runId,
per_page: 100,
});
}
function findResultArtifact(artifacts) {
return artifacts.find((artifact) => artifact.name === ARTIFACT_NAME && !artifact.expired);
}
async function resolve({ github, context, core }) {
const source = context.payload.workflow_run;
const { owner, repo } = context.repo;
if (source.event !== "pull_request") {
core.setOutput("publish", "false");
return;
}
let pullNumber = source.pull_requests?.[0]?.number;
if (!pullNumber) {
let associated;
try {
associated = await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, {
owner,
repo,
commit_sha: source.head_sha,
per_page: 100,
});
} catch (error) {
// A force-pushed or deleted PR head can disappear before this workflow runs.
if (
error.status === 422 &&
error.response?.data?.message === `No commit found for SHA: ${source.head_sha}`
) {
core.info(`Skipping CI result for unavailable commit ${source.head_sha}.`);
core.setOutput("publish", "false");
return;
}
throw error;
}
const matchingPulls = associated.filter(
(pull) =>
pull.state === "open" &&
pull.head.sha === source.head_sha &&
pull.head.ref === source.head_branch,
);
if (matchingPulls.length !== 1) {
core.info(
`Expected one open pull request for ${source.head_repository?.full_name ?? "unknown repository"}:${source.head_branch ?? "unknown branch"} at ${source.head_sha}; found ${matchingPulls.length}.`,
);
core.setOutput("publish", "false");
return;
}
pullNumber = matchingPulls[0].number;
}
if (!pullNumber) {
core.info("No open pull request is associated with the completed CI run.");
core.setOutput("publish", "false");
return;
}
const { data: pull } = await github.rest.pulls.get({ owner, repo, pull_number: pullNumber });
if (pull.head.sha !== source.head_sha) {
core.info(`Skipping stale CI result ${source.head_sha}; PR head is ${pull.head.sha}.`);
core.setOutput("publish", "false");
return;
}
const sourceArtifacts = await artifactsForRun(github, owner, repo, source.id);
const sourceArtifact = findResultArtifact(sourceArtifacts);
// Successful server tests require this artifact; its absence means CI skipped them.
if (!sourceArtifact && source.conclusion === "success") {
core.info("CI skipped the server tests for this change; nothing to report.");
core.setOutput("publish", "false");
return;
}
// One page each for the exact base and the newest runs; paging through
// every successful main run costs up to ten requests to keep 20.
const listRuns = async (filters) => {
const { data } = await github.rest.actions.listWorkflowRuns({
owner,
repo,
workflow_id: source.workflow_id,
branch: pull.base.ref,
event: "push",
status: "success",
per_page: 20,
...filters,
});
return data.workflow_runs;
};
const [baseRuns, recentRuns] = await Promise.all([
listRuns({ head_sha: pull.base.sha }),
listRuns({}),
]);
const orderedRuns = [
...baseRuns,
...recentRuns.filter((run) => run.head_sha !== pull.base.sha),
].slice(0, 20);
let baselineRun;
for (const run of orderedRuns) {
const artifacts = await artifactsForRun(github, owner, repo, run.id);
if (findResultArtifact(artifacts)) {
baselineRun = run;
break;
}
}
core.setOutput("publish", "true");
core.setOutput("pull_number", String(pullNumber));
core.setOutput("pr_artifact", sourceArtifact ? "true" : "false");
core.setOutput("pr_run_id", String(source.id));
core.setOutput("pr_sha", source.head_sha);
core.setOutput("pr_conclusion", source.conclusion ?? "unknown");
core.setOutput("baseline_artifact", baselineRun ? "true" : "false");
core.setOutput("baseline_run_id", baselineRun ? String(baselineRun.id) : "");
core.setOutput("baseline_sha", baselineRun?.head_sha ?? "");
core.setOutput(
"baseline_matches_base",
baselineRun?.head_sha === pull.base.sha ? "true" : "false",
);
}
async function upsertComment(github, context, pullNumber, body, options = {}) {
const { owner, repo } = context.repo;
const comments = await github.paginate(github.rest.issues.listComments, {
owner,
repo,
issue_number: pullNumber,
per_page: 100,
});
const existing = comments.find(
(comment) =>
comment.user?.login === "github-actions[bot]" && comment.body?.includes(COMMENT_MARKER),
);
if (
options.preserveResultSha &&
existing?.body?.includes(resultShaMarker(options.preserveResultSha))
) {
return;
}
if (existing) {
await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
} else {
await github.rest.issues.createComment({ owner, repo, issue_number: pullNumber, body });
}
}
async function upsertCommentForCurrentHead(
github,
context,
core,
pullNumber,
expectedSha,
body,
options,
) {
const { owner, repo } = context.repo;
const { data: pull } = await github.rest.pulls.get({
owner,
repo,
pull_number: pullNumber,
});
if (pull.head.sha !== expectedSha) {
core.info(`Skipping stale CI result ${expectedSha}; PR head is ${pull.head.sha}.`);
return false;
}
await upsertComment(github, context, pullNumber, body, options);
return true;
}
async function publish({ github, context, core }) {
const pullNumber = Number(process.env.PR_NUMBER);
if (!Number.isSafeInteger(pullNumber) || pullNumber <= 0) {
throw new Error("PR_NUMBER is invalid");
}
const current = readResult(process.env.PR_RESULT_DIR);
const currentRun = {
sha: process.env.PR_SHA,
conclusion: process.env.PR_CONCLUSION,
url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.PR_RUN_ID}`,
};
if (!current) {
await upsertCommentForCurrentHead(
github,
context,
core,
pullNumber,
currentRun.sha,
[
COMMENT_MARKER,
"## Thread transfer impact",
"",
`⚠️ The latest [CI run](${currentRun.url}) did not produce a thread transfer result for \`${currentRun.sha.slice(0, 7)}\`.`,
"",
"_This comment will update automatically after the next completed run._",
].join("\n"),
{ preserveResultSha: currentRun.sha },
);
return;
}
const baseline = readResult(process.env.BASELINE_RESULT_DIR);
const baselineRun = baseline
? {
sha: process.env.BASELINE_SHA,
matchesBase: process.env.BASELINE_MATCHES_BASE === "true",
url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.BASELINE_RUN_ID}`,
}
: undefined;
const body = renderComment({ current, baseline, currentRun, baselineRun });
const published = await upsertCommentForCurrentHead(
github,
context,
core,
pullNumber,
currentRun.sha,
body,
);
if (published) {
core.info(`Updated thread transfer report on PR #${pullNumber}.`);
}
}
module.exports = {
publish,
readResult,
renderComment,
resolve,
upsertCommentForCurrentHead,
validateResult,
};