#!/usr/bin/env bun
/**
* Cross-version test suite for alchemy.
*
* Deploys a tiny app (a Cloudflare state store + one Worker) using a range of
* alchemy versions, each from its own folder under ./test, and runs a set of
* upgrade scenarios BACK TO BACK (sequentially, never in parallel — they all
* share one account-wide state store and one fixed worker name):
*
* TEST 1 — sequential 1-by-1 upgrade
* Deploy the oldest version, then upgrade the SAME app in place through
* every version in order (v4 → v5 → v6 → v7 → current), walking the
* account-wide state store up one version at a time.
*
* TEST 2 — direct-to-latest
* For each older example, deploy it and then upgrade the SAME app DIRECTLY
* to latest (the current branch), skipping the intermediate versions — a
* big-jump upgrade (e.g. state store v4 → v7 in one step).
*
* Both tests assert, after every deploy, that the live worker serves the marker
* baked into the version that was just deployed — proving the running code was
* actually replaced in place.
*
* Each scenario ("unit") owns a FRESH state store: the runner tears the state
* store (worker + secrets) down before and after every unit, so — e.g. — the
* 1-by-1 upgrade runs on its own store, that store is destroyed, then each
* direct-to-latest jump deploys a brand-new store. (Pass --reuse-store to skip
* the teardowns and reuse one store across units, which is faster but less
* isolated.)
*
* Stage folders (see ./test/*):
* 01-beta.39 — last npm release with state store version 4
* 02-beta.44 — last npm release with state store version 5
* 03-beta.45 — last npm release with state store version 6
* 04-beta.59 — last v2 release on npm (state store version 7 = latest npm)
* 05-current — the current branch (workspace source, state store version 7)
*
* ──────────────────────────────────────────────────────────────────────────
* WARNING: the Cloudflare state store is ACCOUNT-WIDE and shared by every
* stack on the account/profile. These tests bootstrap state store v4, which
* DOWNGRADES the store if the account already had a higher version. Run this
* against a dedicated / throwaway Cloudflare account only.
* ──────────────────────────────────────────────────────────────────────────
*
* Usage:
* bun run test/cross-version-test/run.ts --profile <profile> [flags]
* ALCHEMY_PROFILE=<profile> bun run test/cross-version-test/run.ts
*
* The suite runs FORCE-THROUGH: every edge in `EDGES` is attempted on its own
* fresh state store, a failure is recorded (never aborts the run), and a
* PASS/FAIL summary + a `===RESULTS_JSON===` block is printed at the end. Mark
* an edge KNOWN BAD by commenting it out of `EDGES` (with a note).
*
* Usage:
* bun run test/cross-version-test/run.ts --profile <profile> [flags]
* ALCHEMY_PROFILE=<profile> bun run test/cross-version-test/run.ts
*
* Flags:
* --profile <p> Cloudflare auth profile (~/.alchemy/profiles.json).
* Falls back to $ALCHEMY_PROFILE. REQUIRED (no default, to
* avoid accidentally mutating the wrong account).
* --group <g> Only run edges in one group: "sequential" or "jump".
* --stage <s> Alchemy stage name for the app. Default: "xver".
* --no-install Skip `bun install` in npm stages (reuse existing installs).
* --reuse-store Reuse ONE state store across all edges (skip the per-edge
* teardown/redeploy). Faster, but edges aren't isolated.
* --settle <sec> Seconds to wait after a store teardown for workers.dev to
* propagate the deletion (default 25). Avoids the fresh
* bootstrap seeing the just-deleted worker's stale version.
* --boot-retries <n> Bootstrap attempts on transient 404/500/version-not-
* ready during hoist (default 3).
*/
import { spawn } from "node:child_process";
import * as fs from "node:fs";
import * as path from "node:path";
import { fileURLToPath } from "node:url";
const HERE = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(HERE, "..", "..");
const TEST_DIR = path.join(HERE, "test");
interface Stage {
dir: string;
label: string;
marker: string;
/** npm stages get an isolated `bun install`; workspace stages don't. */
kind: "npm" | "workspace";
}
const STAGES: Stage[] = [
{
dir: "01-beta.39",
label: "alchemy@2.0.0-beta.39 (last npm release with state store v4)",
marker: "01-beta.39",
kind: "npm",
},
{
dir: "02-beta.44",
label: "alchemy@2.0.0-beta.44 (last npm release with state store v5)",
marker: "02-beta.44",
kind: "npm",
},
{
dir: "03-beta.45",
label: "alchemy@2.0.0-beta.45 (last npm release with state store v6)",
marker: "03-beta.45",
kind: "npm",
},
{
dir: "04-beta.59",
label: "alchemy@2.0.0-beta.59 (latest v2 on npm, state store v7)",
marker: "04-beta.59",
kind: "npm",
},
{
dir: "05-current",
label: "current branch (workspace source, state store v7)",
marker: "05-current",
kind: "workspace",
},
];
// The newest version — the target every TEST 2 jump upgrades directly to.
const LATEST = STAGES[STAGES.length - 1];
// ── arg parsing ───────────────────────────────────────────────────────────
function parseArgs(argv: string[]) {
const out: Record<string, string | boolean> = {};
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (!a.startsWith("--")) continue;
const key = a.slice(2);
const next = argv[i + 1];
if (next === undefined || next.startsWith("--")) {
out[key] = true;
} else {
out[key] = next;
i++;
}
}
return out;
}
const args = parseArgs(process.argv.slice(2));
const PROFILE = (args.profile as string) ?? process.env.ALCHEMY_PROFILE;
const STAGE = (args.stage as string) ?? "xver";
const NO_INSTALL = args["no-install"] === true;
const REUSE_STORE = args["reuse-store"] === true;
// Seconds to wait after a state-store teardown before redeploying, so the
// same-named worker's DELETION propagates on workers.dev (otherwise the fresh
// bootstrap sees the old worker's version — "version not ready").
const SETTLE = Number(args.settle ?? 25);
// Retries for the bootstrap step, which can transiently 404/500 while the
// freshly-deployed store worker's HTTP endpoint comes up.
const BOOT_RETRIES = Number(args["boot-retries"] ?? 3);
const GROUP = typeof args.group === "string" ? (args.group as "sequential" | "jump") : undefined;
const RESET = "\x1b[0m";
const BOLD = "\x1b[1m";
const GREEN = "\x1b[32m";
const RED = "\x1b[31m";
const YELLOW = "\x1b[33m";
const CYAN = "\x1b[36m";
function banner(msg: string) {
const line = "─".repeat(Math.min(78, msg.length + 4));
console.log(`\n${CYAN}${line}\n ${BOLD}${msg}${RESET}${CYAN}\n${line}${RESET}`);
}
const sleep = (seconds: number) => new Promise((r) => setTimeout(r, seconds * 1000));
if (!PROFILE) {
console.error(
`${RED}error: no Cloudflare profile.${RESET}\n` +
`Pass --profile <name> or set $ALCHEMY_PROFILE.\n` +
`This run mutates the account-wide state store — point it at a\n` +
`dedicated/throwaway account, not your shared testing account.`,
);
process.exit(2);
}
// ── subprocess helper: stream output live AND capture it ────────────────────
function run(
cmd: string,
cmdArgs: string[],
opts: { cwd: string },
): Promise<{ code: number; output: string }> {
return new Promise((resolve) => {
const printable = `${cmd} ${cmdArgs.join(" ")}`;
console.log(`${YELLOW}$ (${path.basename(opts.cwd)}) ${printable}${RESET}`);
const child = spawn(cmd, cmdArgs, {
cwd: opts.cwd,
// CI=true makes the state-store version check FAIL FAST (die) on a
// mismatch instead of prompting `Yes/No` — which would hang forever on a
// non-TTY subprocess. `--yes` only covers plan approval, not this prompt.
env: { ...process.env, ALCHEMY_PROFILE: PROFILE, CI: "true" },
shell: process.platform === "win32", // resolve `bun`/.cmd on Windows
// No stdin: any prompt that slips past CI=true reads EOF and cancels
// rather than blocking forever on a non-TTY pipe.
stdio: ["ignore", "pipe", "pipe"],
});
let output = "";
const onData = (buf: Buffer, to: NodeJS.WriteStream) => {
const text = buf.toString();
output += text;
to.write(text);
};
child.stdout.on("data", (b) => onData(b, process.stdout));
child.stderr.on("data", (b) => onData(b, process.stderr));
child.on("close", (code) => resolve({ code: code ?? 1, output }));
child.on("error", (err) => {
output += String(err);
console.error(`${RED}spawn error: ${err}${RESET}`);
resolve({ code: 1, output });
});
});
}
const stageDir = (stage: Stage) => path.join(TEST_DIR, stage.dir);
/**
* Clear the state-store bootstrap's LOCAL staging state
* (`<stageDir>/.alchemy/state/CloudflareStateStore`). Bootstrap deploys the
* state store into this local backend first, then hoists it to the remote store
* and deletes it — but a run killed mid-bootstrap leaves it behind, so the next
* bootstrap "resumes" it against cloud resources we've since torn down. Wiping
* it makes every bootstrap start fresh. (Only the bootstrap uses local state;
* the app itself uses the remote store, so this is safe.)
*/
function clearLocalBootstrapState(stage: Stage) {
const dir = path.join(stageDir(stage), ".alchemy", "state", "CloudflareStateStore");
try {
fs.rmSync(dir, { recursive: true, force: true });
} catch {
/* ignore */
}
}
// `bun run alc -- <subcommand...>` runs the folder's alchemy CLI:
// - npm stages -> the version pinned in that folder's node_modules
// - workspace stage -> the workspace `alchemy` bin (current branch source),
// resolved by walking up to the repo-root node_modules
function alc(stage: Stage, subArgs: string[]) {
return run("bun", ["run", "alc", "--", ...subArgs], { cwd: stageDir(stage) });
}
function extractUrl(output: string): string | undefined {
// Strip ANSI so the regex matches output rendered through the CLI reporter.
// oxlint-disable-next-line no-control-regex
const clean = output.replace(/\x1b\[[0-9;]*m/g, "");
const match = clean.match(/https:\/\/[a-z0-9.-]+\.workers\.dev[^\s"')]*/i);
return match?.[0];
}
// Pull a concise failure reason out of a subprocess's output for the summary.
// Skips known benign noise (the CLI's tsconfig warning, the upgrade nag) so it
// doesn't mask the real error — and so the transient-retry check sees the truth.
const NOISE = /tsconfig|is available|npm_|Run `bun add`/i;
const MEANINGFUL =
/StateStoreError|BadRequest|AuthError|Decode error|Transport error|HttpClientError|ERROR \(#|not found|Unauthorized|Forbidden|version not ready/i;
function extractError(output: string): string {
// oxlint-disable-next-line no-control-regex
const clean = output.replace(/\x1b\[[0-9;]*m/g, "");
const lines = clean
.split("\n")
.map((l) => l.trim())
.filter(Boolean);
// Prefer the CLI's structured "ERROR (#N): <message>" line — that's the real
// cause. Falling back to the raw `_tag:`/`[cause]:` object-dump lines both
// misreports AND defeats the transient-retry check.
for (const l of lines) {
const m = l.match(/ERROR \(#\d+\):\s*(.+)/);
if (m?.[1] && !NOISE.test(m[1])) return m[1].slice(0, 240);
}
const line = lines.find(
(l) => MEANINGFUL.test(l) && !NOISE.test(l) && !/^_tag:|^\[cause\]:|^"~effect/.test(l),
);
return (line ?? "unknown error").slice(0, 240);
}
// Transient state-store bootstrap/deploy failures that clear once the freshly
// (re)deployed worker propagates on workers.dev — retry these.
const TRANSIENT =
/Decode error|not ready|version not ready|not found|Transport error|\b404\b|\b500\b|\b502\b|\b503\b|fetch failed|ECONN|ETIMEDOUT/i;
async function verify(url: string, expectedMarker: string): Promise<void> {
// Fresh / freshly-updated workers.dev routes transiently 404/5xx while the
// edge propagates, so retry with backoff.
let lastErr: unknown;
for (let attempt = 1; attempt <= 12; attempt++) {
try {
const res = await fetch(url);
if (res.ok) {
const body = (await res.json()) as { marker?: string };
if (body.marker === expectedMarker) {
console.log(`${GREEN}✓ live worker serves marker "${body.marker}"${RESET}`);
return;
}
throw new Error(`marker mismatch: got "${body.marker}", expected "${expectedMarker}"`);
}
lastErr = new Error(`HTTP ${res.status}`);
} catch (err) {
lastErr = err;
}
await new Promise((r) => setTimeout(r, Math.min(1000 * attempt, 5000)));
}
throw new Error(
`verification failed for ${url} (expected marker "${expectedMarker}"): ${lastErr instanceof Error ? lastErr.message : String(lastErr)}`,
);
}
// ── stage primitives ────────────────────────────────────────────────────────
const installed = new Set<string>();
async function ensureInstalled(stage: Stage) {
if (stage.kind !== "npm" || NO_INSTALL || installed.has(stage.dir)) return;
const r = await run("bun", ["install"], { cwd: stageDir(stage) });
if (r.code !== 0) throw new Error(`install failed for ${stage.dir}`);
installed.add(stage.dir);
}
/**
* Run an alchemy CLI subcommand, retrying on TRANSIENT state-store failures
* (404/500 hoisting into a freshly-deployed worker, stale version, transport
* blips → "store not found"). A non-transient failure (e.g. the real v5→v6
* `500 GET … DecodeError` read incompat) is thrown on the first attempt-set.
*/
async function alcRetry(
stage: Stage,
argv: string[],
what: string,
): Promise<{ code: number; output: string }> {
let err = "";
for (let attempt = 1; attempt <= Math.max(1, BOOT_RETRIES); attempt++) {
if (what === "bootstrap") clearLocalBootstrapState(stage); // never resume a stale local stack
const r = await alc(stage, argv);
if (r.code === 0) return r;
err = extractError(r.output);
if (attempt < BOOT_RETRIES && TRANSIENT.test(err)) {
console.log(
`${YELLOW}${what} attempt ${attempt}/${BOOT_RETRIES} failed (${err}) — settling ${SETTLE}s and retrying${RESET}`,
);
await sleep(SETTLE);
continue;
}
break;
}
throw new Error(`${what} failed for ${stage.dir}: ${err}`);
}
/**
* Deploy (or in-place upgrade) the app to `stage`'s version under the given
* alchemy stage, and assert the live worker serves that version's marker.
* Bootstraps the account-wide state store to the version's STATE_STORE_VERSION
* first (idempotent; handles both up- and down-grades), which keeps the deploy
* non-interactive. Both bootstrap and deploy retry on transient errors.
*/
async function deployAndVerify(stage: Stage, alchemyStage: string): Promise<void> {
await ensureInstalled(stage);
const cloudflareCommand =
stage.kind === "workspace" ? ["provider", "cloudflare"] : ["cloudflare"];
await alcRetry(stage, [...cloudflareCommand, "bootstrap", "--profile", PROFILE!], "bootstrap");
const dep = await alcRetry(
stage,
// --adopt: take over a pre-existing fixed-name worker instead of failing.
["deploy", "--yes", "--adopt", "--stage", alchemyStage, "--profile", PROFILE!],
"deploy",
);
const url = extractUrl(dep.output);
if (!url) {
throw new Error(`could not determine worker URL for ${stage.dir}`);
}
console.log(`worker url: ${url}`);
await verify(url, stage.marker);
}
async function destroyApp(stage: Stage, alchemyStage: string) {
const r = await alc(stage, ["destroy", "--yes", "--stage", alchemyStage, "--profile", PROFILE!]);
if (r.code !== 0) {
console.error(
`${RED}warning: destroy failed (stage ${alchemyStage}) — clean up manually.${RESET}`,
);
} else {
console.log(`${GREEN}✓ app destroyed (stage ${alchemyStage})${RESET}`);
}
}
/**
* Delete the account-wide Cloudflare state store (worker + secrets store) via
* the current-branch CLI's `provider cloudflare teardown` (idempotent). Run
* between units so each one deploys a FRESH state store rather than inheriting
* the previous unit's (possibly downgraded) one.
*/
async function teardownStore(reason: string) {
console.log(`${YELLOW}↺ tearing down state store (${reason})${RESET}`);
const r = await alc(LATEST, ["provider", "cloudflare", "teardown", "--profile", PROFILE!]);
if (r.code !== 0) {
console.error(`${RED}warning: state store teardown failed — remove it manually.${RESET}`);
}
// Let the worker DELETION propagate on workers.dev before anything redeploys
// the same-named worker — otherwise the fresh bootstrap sees the old version.
if (SETTLE > 0) {
console.log(`${YELLOW} settling ${SETTLE}s for edge propagation…${RESET}`);
await sleep(SETTLE);
}
}
const stageByDir = (dir: string): Stage => {
const s = STAGES.find((x) => x.dir === dir);
if (!s) throw new Error(`unknown stage dir: ${dir}`);
return s;
};
// ── The upgrade paths under test ─────────────────────────────────────────────
// Each edge = "deploy `from`, then upgrade the same app to `to`" on its OWN
// fresh state store. The suite runs EVERY edge independently and force-through
// (a failure is recorded, never aborts the run). Comment an edge out (and note
// why) to mark it KNOWN BAD and skip it.
interface Edge {
from: string; // stage dir
to: string; // stage dir
group: "sequential" | "jump";
}
const EDGES: Edge[] = [
// ── sequential 1-by-1 upgrade steps (npm → npm) ──
{ from: "01-beta.39", to: "02-beta.44", group: "sequential" }, // v4 → v5
// KNOWN BAD (v5 → v6): after the store is upgraded to v6 (beta.45), reading a
// record written under ≤v5 fails with `500 GET … DecodeError`. beta.45's
// legacy-record read (the createdAt/updatedAt reshape, PR #427) is broken;
// it's fixed in v7 — upgrading v5 straight to current works (see the v5 →
// worktree jump). So don't step through beta.45 with pre-v6 state; skip to
// beta.46+.
// { from: "02-beta.44", to: "03-beta.45", group: "sequential" }, // v5 → v6
{ from: "03-beta.45", to: "04-beta.59", group: "sequential" }, // v6 → v7
{ from: "04-beta.59", to: "05-current", group: "sequential" }, // v7 → worktree
// ── direct-to-latest jumps (→ worktree, skipping the middle) ──
// KNOWN BAD (v4 → worktree): the current state-store client can't write to a
// v4-format store — upgrading it fails with `415 Unsupported Media Type` on
// `PUT …/StateStoreEncryptionKey` (deterministic; fails every retry). v4
// (beta.37–39) predates the RPC state-store rewrite at v5, so its store HTTP
// API is wire-incompatible with current. v5/v6/v7 → worktree all work, so
// step a v4 store up to ≥v5 before jumping to latest.
// { from: "01-beta.39", to: "05-current", group: "jump" }, // v4 → worktree
{ from: "02-beta.44", to: "05-current", group: "jump" }, // v5 → worktree
{ from: "03-beta.45", to: "05-current", group: "jump" }, // v6 → worktree
];
const EDGE_STAGE = STAGE; // one alchemy stage; each edge gets a fresh store anyway
interface StepResult {
name: string;
status: "ok" | "fail" | "skip";
error?: string;
}
interface EdgeResult {
label: string;
group: string;
worktree: boolean;
steps: StepResult[];
failed: boolean;
}
async function tryStep(name: string, fn: () => Promise<void>): Promise<StepResult> {
try {
await fn();
console.log(`${GREEN}✓ ${name}${RESET}`);
return { name, status: "ok" };
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
console.error(`${RED}✗ ${name}: ${msg}${RESET}`);
return { name, status: "fail", error: msg };
}
}
async function runEdge(edge: Edge): Promise<EdgeResult> {
const from = stageByDir(edge.from);
const to = stageByDir(edge.to);
const worktree = from.kind === "workspace" || to.kind === "workspace";
const label = `${from.dir} → ${to.dir}`;
banner(`EDGE ${label} [${edge.group}${worktree ? ", worktree" : ""}]`);
// Fresh store for this edge.
clearLocalBootstrapState(from);
clearLocalBootstrapState(to);
if (!REUSE_STORE) await teardownStore(`fresh for ${label}`);
const steps: StepResult[] = [];
const deployStep = await tryStep(`deploy ${from.dir}`, () => deployAndVerify(from, EDGE_STAGE));
steps.push(deployStep);
if (deployStep.status === "ok") {
steps.push(await tryStep(`upgrade → ${to.dir}`, () => deployAndVerify(to, EDGE_STAGE)));
} else {
steps.push({
name: `upgrade → ${to.dir}`,
status: "skip",
error: "prerequisite deploy failed",
});
}
// Best-effort cleanup so the next edge starts clean.
await destroyApp(to, EDGE_STAGE).catch(() => {});
if (!REUSE_STORE) await teardownStore(`after ${label}`).catch(() => {});
return {
label,
group: edge.group,
worktree,
steps,
failed: steps.some((s) => s.status !== "ok"),
};
}
// ── main ────────────────────────────────────────────────────────────────────
async function main() {
const edges = EDGES.filter((e) => !GROUP || e.group === GROUP);
banner("alchemy cross-version test suite (force-through)");
console.log(`profile : ${PROFILE}`);
console.log(`edges : ${edges.length}`);
console.log(`store : ${REUSE_STORE ? "reused" : "FRESH per edge"}`);
console.log(`repo : ${REPO_ROOT}`);
console.log(
`${YELLOW}note: bootstraps state store v4 — run against a dedicated account.${RESET}`,
);
if (!REUSE_STORE) await teardownStore("pre-clean");
const results: EdgeResult[] = [];
for (const edge of edges) {
results.push(await runEdge(edge));
}
// ── summary ────────────────────────────────────────────────────────────--
banner("SUMMARY");
for (const r of results) {
const mark = r.failed ? `${RED}FAIL${RESET}` : `${GREEN}PASS${RESET}`;
const tag = r.worktree ? " (worktree)" : "";
console.log(`${mark} [${r.group}] ${r.label}${tag}`);
for (const s of r.steps) {
const sym =
s.status === "ok"
? `${GREEN}✓${RESET}`
: s.status === "skip"
? `${YELLOW}∅${RESET}`
: `${RED}✗${RESET}`;
console.log(` ${sym} ${s.name}${s.error ? ` — ${s.error}` : ""}`);
}
}
// Machine-readable block for downstream parsing.
console.log("\n===RESULTS_JSON===");
console.log(
JSON.stringify(
results.map((r) => ({
label: r.label,
group: r.group,
worktree: r.worktree,
failed: r.failed,
steps: r.steps.map((s) => ({
name: s.name,
status: s.status,
error: s.error,
})),
})),
null,
2,
),
);
console.log("===END_RESULTS_JSON===");
const failed = results.filter((r) => r.failed);
banner(
failed.length === 0
? `${GREEN}ALL EDGES PASSED${RESET}`
: `${RED}${failed.length}/${results.length} EDGES FAILED${RESET}`,
);
}
main().catch((err) => {
console.error(`\n${RED}${BOLD}suite crashed:${RESET} ${err}`);
process.exit(1);
});