import { randomBytes, createHash } from "node:crypto";
import * as Docker from "@alchemy.run/docker";
import * as Fly from "@alchemy.run/fly";
import { AppDeletionAmbiguous } from "@alchemy.run/fly/App";
import { readinessRoles, DeploymentRecoveryAmbiguous } from "@alchemy.run/fly/bluegreen";
import { makeMachineLeases } from "@alchemy.run/fly/leases";
import { type MachineContainer, type MachineProps } from "@alchemy.run/fly/Machine";
import { alchemyMetadataKeys as keys } from "@alchemy.run/fly/Metadata";
import {
waitHealthy,
autostopMode,
checksPassing,
ensureStarted,
observeReplicaSet,
predecessorShutdown,
deleteReplicaSet,
ReplicaNotCreated,
ReplicaRetirementIncomplete,
retireMachines,
retireMachine,
} from "@alchemy.run/fly/replicas";
import { ReplicaChecksNotPassing } from "@alchemy.run/fly/replicas";
import { expect, assert, it, describe } from "@alchemy.run/test";
import * as Test from "@alchemy.run/test/Alchemy";
import type { ScratchStack } from "@alchemy.run/test/Alchemy";
import * as TestCore from "@alchemy.run/test/Core";
import { scratchStack, withProviders } from "@alchemy.run/test/Core";
import {
GatewayTimeout,
HTTP_STATUS_MAP,
RETRYABLE_HTTP_STATUSES,
} from "@distilled.cloud/core/errors";
import { Credentials } from "@distilled.cloud/fly-io/Credentials";
import { GatewayTimeout as GatewayTimeoutErrors } from "@distilled.cloud/fly-io/Errors";
import * as machines from "@distilled.cloud/fly-io/machines";
import { type Machine } from "@distilled.cloud/fly-io/machines";
import * as Retry from "@distilled.cloud/fly-io/Retry";
import * as Alchemy from "alchemy";
import { DestroyError } from "alchemy/Apply";
import { localState, makeLocalState } from "alchemy/State/LocalState";
import * as Cause from "effect/Cause";
import * as Clock from "effect/Clock";
import * as ConfigProvider from "effect/ConfigProvider";
import * as Data from "effect/Data";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as Fiber from "effect/Fiber";
import * as FileSystem from "effect/FileSystem";
import * as FetchHttpClient from "effect/http/FetchHttpClient";
import * as HttpClient from "effect/http/HttpClient";
import * as HttpClientRequest from "effect/http/HttpClientRequest";
import * as Path from "effect/Path";
import * as Redacted from "effect/Redacted";
import * as Ref from "effect/Ref";
import * as Result from "effect/Result";
import * as Schedule from "effect/Schedule";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";
import { engineActor } from "./fixtures/actors.ts";
import { dropCompletedCreate } from "./fixtures/bluegreen-create-proxy.ts";
import {
Site,
Token,
TRIGGER_SECRET,
Writer,
writerLayer,
} from "./fixtures/bluegreen-runtime-secrets/writer.ts";
import {
BoundSecrets,
CacheOne,
CacheTwo,
Site as SiteBluegreenSecrets,
} from "./fixtures/bluegreen-secrets.ts";
import {
assertOrder,
assertReplacement,
assertStopped,
makeScenario,
requireValue,
} from "./fixtures/bluegreen-worker-test.ts";
import {
assertAppGone,
census,
checks,
deployWorker,
assertCommitted,
} from "./fixtures/bluegreen.ts";
import { sanitizeExecFailure } from "./fixtures/exec-lease.ts";
import { makeReadinessControl, repairReadiness } from "./fixtures/http-readiness-control.ts";
import {
assertReadinessCommit,
readinessActor,
readinessChecksPassing,
readinessProxy,
retires,
type ReadinessEvent,
} from "./fixtures/idle-cadence-readiness.ts";
import {
observeStops,
type StopRequest,
writeLegacyProtocol,
} from "./fixtures/legacy-protocol-writer.ts";
import MountedBlueGreen from "./fixtures/mounted-bluegreen.ts";
import {
assertBarrierInventory,
assertClean,
assertConverged,
assertSingleRunner,
census as censusProcessDeath,
deploy,
evidencePaths,
heldLeases,
identity,
machine,
matchesBarrier,
nowSeconds,
observeLeaseExpiry,
persistedRow,
phases,
processDeathFile,
readWitness,
writeEvidence,
type Phase,
type Witness,
} from "./fixtures/process-death.ts";
import {
appName,
candidateId,
MachineServiceAutostop,
metadata,
protocolClient,
reconcile,
reply,
withControlledClient,
} from "./fixtures/protocol-branches.ts";
import {
Finalized,
RunnerInterrupted,
Witness as WitnessSignalOverlap,
assertBoundary,
assertConverged as assertConvergedSignalOverlap,
assertInventory,
assertReleased,
assertSingleRunner as assertSingleRunnerSignalOverlap,
boundary,
cases,
firstReturnedUncordon,
matches,
observeExpiry,
observeLeases,
pathsFor,
readEvidence,
signalOverlapFile,
successful,
type SignalCase,
} from "./fixtures/signal-overlap.ts";
import { delayedResourceWrite, ResourceRow } from "./fixtures/state-persistence.ts";
import { transportProxy, throughProxy, type TransportEvent } from "./fixtures/transport.ts";
// Blocks are independent (each case deploys its own stack). Fully parallel
// overloads Fly's API (slow App deletes, missed timing windows), so they run
// in six concurrent shards balanced by measured duration (~35 min each);
// blocks run one at a time within a shard, and process-wide cases keep
// `exclusive: true`.
/** URL param values may be `Redacted` (sent unwrapped); compare their plain text. */
const unredact = (value: string | Redacted.Redacted | undefined) =>
Redacted.isRedacted(value) ? String(Redacted.value(value)) : value;
const decodeAutostop = Schema.decodeUnknownSync(MachineServiceAutostop);
describe.concurrent("BlueGreen", () => {
describe.sequential("shard 1", () => {
describe.sequential(
"idle topology",
{
tags: [
"provider:fly",
"provider:fly:app",
"provider:fly:machine",
"provider:fly:service",
"live",
],
},
() => {
const file = "test/Fly/BlueGreen.test.ts";
const { test } = Test.make({ providers: Fly.providers() });
type IdleMode = "stop" | "suspend";
const topology = (
mode: IdleMode,
count: number,
floor = 0,
mixed = false,
): Partial<Omit<Extract<MachineProps, { image: string }>, "app">> => ({
count,
deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" },
services: [
{
protocol: "tcp",
internalPort: 80,
autostop: mode,
autostart: true,
minMachinesRunning: floor,
checks: [checks.ready],
},
...(mixed ? [{ protocol: "tcp", internalPort: 81, autostop: "off" as const }] : []),
],
});
const waitState = (appName: string, machineId: string, state: string) =>
machines.getMachine({ app_name: appName, machine_id: machineId }).pipe(
Effect.repeat({
schedule: Schedule.spaced("2 seconds"),
times: 30,
until: (machine) => machine.state === state,
}),
Effect.tap((machine) => Effect.sync(() => expect(machine.state).toBe(state))),
Effect.timeout("90 seconds"),
);
// Native stop/suspend is setup only; automatic return-to-idle has its own test.
const idleAll = (appName: string, ids: string[], mode: IdleMode) =>
Effect.forEach(ids, (id) =>
Effect.gen(function* () {
const target = { app_name: appName, machine_id: id };
const current = yield* machines.getMachine(target);
if (current.state !== "started") {
yield* machines.startMachine(target);
yield* waitState(appName, id, "started");
}
if (mode === "stop") {
yield* machines.stopMachine({ ...target, signal: "SIGQUIT", timeout: "5s" });
} else {
yield* machines.suspendMachine(target);
}
yield* waitState(appName, id, mode === "stop" ? "stopped" : "suspended");
}),
);
// Expected slots are literal scenario inputs, never computed by readinessRoles.
const assertSlots = (live: machines.Machine[], runningSlots: readonly number[]) => {
const ordered = [...live].sort(
(left, right) =>
Number(left.config?.metadata?.["alchemy.replica"]) -
Number(right.config?.metadata?.["alchemy.replica"]),
);
expect(ordered.map((machine) => machine.config?.metadata?.["alchemy.replica"])).toEqual(
Array.from({ length: live.length }, (_, index) => String(index)),
);
expect(
ordered
.filter((machine) => machine.config?.metadata?.["alchemy.readiness-role"] === "run")
.map((machine) => Number(machine.config?.metadata?.["alchemy.replica"])),
).toEqual([...runningSlots]);
const roles = Array.from({ length: live.length }, (_, slot) =>
runningSlots.includes(slot) ? "run" : "idle",
).join(",");
for (const [slot, machine] of ordered.entries()) {
expect(machine.config?.metadata?.["alchemy.readiness-role"]).toBe(
runningSlots.includes(slot) ? "run" : "idle",
);
expect(machine.config?.metadata?.["alchemy.readiness-roles"]).toBe(roles);
}
return ordered;
};
const assertTopology = (
appName: string,
ids: string[],
mode: IdleMode,
runningSlots: readonly number[],
floor = 0,
mixed = false,
) =>
Effect.gen(function* () {
const live = yield* assertCommitted(appName, ids);
const ordered = assertSlots(live, runningSlots);
expect(
new Set(live.map((machine) => machine.config?.metadata?.["alchemy.generation"])).size,
).toBe(1);
for (const [slot, machine] of ordered.entries()) {
expect(machine.config?.metadata?.["alchemy.idle-policy-restored"]).toBe("true");
expect(machine.cordoned).toBe(false);
expect(machine.config?.services).toHaveLength(mixed ? 2 : 1);
expect(autostopMode(machine.config?.services?.[0]?.autostop)).toBe(mode);
expect(machine.config?.services?.[0]?.autostart).toBe(true);
expect(machine.config?.services?.[0]?.min_machines_running).toBe(floor);
if (mixed) {
expect(autostopMode(machine.config?.services?.[1]?.autostop)).toBe("off");
}
if (!runningSlots.includes(slot)) {
expect(["created", "stopped", "suspended"]).toContain(machine.state);
} else {
expect(mixed ? ["started"] : ["started", "stopped", "suspended"]).toContain(
machine.state,
);
expect(machine.instance_id).toBeDefined();
expect(machine.config?.metadata?.["alchemy.checked-instance"]).toBe(
machine.instance_id,
);
}
}
return ordered;
});
const checkedDeployment = (
actor: Test.ScratchStack,
proxy: { events: TransportEvent[]; readiness: ReadinessEvent[] },
version: string,
mode: IdleMode,
count: number,
runningSlots: readonly number[],
priorIds: readonly string[] = [],
floor = 0,
mixed = false,
) =>
Effect.gen(function* () {
const begin = proxy.events.length;
const proofBegin = proxy.readiness.length;
const output = yield* deployWorker(
actor,
version,
topology(mode, count, floor, mixed),
).pipe(Effect.scoped);
const live = yield* assertTopology(
output.appName,
output.machineIds,
mode,
runningSlots,
floor,
mixed,
);
assertReadinessCommit(
proxy.readiness.slice(proofBegin),
priorIds,
live,
runningSlots,
["ready", "servicecheck-00-http-80"],
!mixed,
);
expect(
proxy.events
.slice(begin)
.some(
(event) =>
event.stage === "request" &&
priorIds.includes(event.machineId!) &&
event.path.endsWith("/start"),
),
).toBe(false);
return output;
});
// Each case deploys its own stack; these are the file's longest cases
// (3-8 min of real idle waits), so run them side by side.
describe.concurrent("live idle topology and automatic cadence", () => {
for (const mode of ["stop", "suspend"] as const) {
test.provider(
`S05 S11 F13 ${mode} real all-idle 1 to 3 to 1 scaling preserves representative-only readiness`,
(stack) =>
Effect.gen(function* () {
yield* stack.destroy();
const site = yield* stack.deploy(Fly.App("Site"));
try {
const proxy = yield* readinessProxy();
const actor = yield* readinessActor(
stack,
`S05 S11 F13 ${mode} real all-idle 1 to 3 to 1 scaling preserves representative-only readiness`,
file,
proxy,
);
const first = yield* checkedDeployment(actor, proxy, "one", mode, 1, [0]);
yield* idleAll(site.appName, first.machineIds, mode);
const up = yield* checkedDeployment(
actor,
proxy,
"two",
mode,
3,
[0],
first.machineIds,
);
expect(up.machineIds).toHaveLength(3);
expect(up.machineIds.every((id) => !first.machineIds.includes(id))).toBe(true);
yield* idleAll(site.appName, up.machineIds, mode);
const down = yield* checkedDeployment(
actor,
proxy,
"three",
mode,
1,
[0],
up.machineIds,
);
expect(down.machineIds).toHaveLength(1);
expect(up.machineIds).not.toContain(down.machineId);
yield* idleAll(site.appName, down.machineIds, mode);
const unchanged = yield* deployWorker(actor, "three", topology(mode, 1));
expect(unchanged.machineIds).toEqual(down.machineIds);
expect((yield* census(site.appName))[0]!.state).toBe(
mode === "stop" ? "stopped" : "suspended",
);
} finally {
yield* stack.destroy();
yield* assertAppGone(site.appName);
}
}).pipe(Effect.scoped),
{ timeout: 900_000 },
);
test.provider(
`S05 S11 F13 ${mode} live floor 0 to 2 to 0 and mixed off-plus-idle service topology`,
(stack) =>
Effect.gen(function* () {
yield* stack.destroy();
const site = yield* stack.deploy(Fly.App("Site"));
try {
const proxy = yield* readinessProxy();
const actor = yield* readinessActor(
stack,
`S05 S11 F13 ${mode} live floor 0 to 2 to 0 and mixed off-plus-idle service topology`,
file,
proxy,
);
const first = yield* checkedDeployment(actor, proxy, "one", mode, 3, [0]);
yield* idleAll(site.appName, first.machineIds, mode);
const raised = yield* checkedDeployment(
actor,
proxy,
"one",
mode,
3,
[0, 1],
first.machineIds,
2,
);
yield* idleAll(site.appName, raised.machineIds, mode);
const lowered = yield* checkedDeployment(
actor,
proxy,
"one",
mode,
3,
[0],
raised.machineIds,
);
yield* idleAll(site.appName, lowered.machineIds, mode);
const slots = assertSlots(yield* census(site.appName), [0]);
yield* machines.startMachine({
app_name: site.appName,
machine_id: slots[2]!.id!,
});
yield* waitState(site.appName, slots[2]!.id!, "started");
const mixedOld = (yield* census(site.appName)).sort(
(a, b) =>
Number(a.config?.metadata?.["alchemy.replica"]) -
Number(b.config?.metadata?.["alchemy.replica"]),
);
expect(mixedOld.map((machine) => machine.state)).toEqual([
mode === "stop" ? "stopped" : "suspended",
mode === "stop" ? "stopped" : "suspended",
"started",
]);
const retained = yield* checkedDeployment(
actor,
proxy,
"retained",
mode,
3,
[2],
lowered.machineIds,
);
yield* idleAll(site.appName, retained.machineIds, mode);
const retainedSlots = assertSlots(yield* census(site.appName), [2]);
yield* machines.startMachine({
app_name: site.appName,
machine_id: retainedSlots[2]!.id!,
});
yield* waitState(site.appName, retainedSlots[2]!.id!, "started");
expect(
assertSlots(yield* census(site.appName), [2]).map((machine) => machine.state),
).toEqual([
mode === "stop" ? "stopped" : "suspended",
mode === "stop" ? "stopped" : "suspended",
"started",
]);
const retainedFloor = yield* checkedDeployment(
actor,
proxy,
"retained",
mode,
3,
[0, 2],
retained.machineIds,
2,
);
yield* idleAll(site.appName, retainedFloor.machineIds, mode);
const mixed = yield* checkedDeployment(
actor,
proxy,
"one",
mode,
3,
[0, 1, 2],
retainedFloor.machineIds,
0,
true,
);
yield* idleAll(site.appName, mixed.machineIds, mode);
yield* checkedDeployment(actor, proxy, "one", mode, 3, [0], mixed.machineIds);
} finally {
yield* stack.destroy();
yield* assertAppGone(site.appName);
}
}).pipe(Effect.scoped),
{ timeout: 1_200_000 },
);
for (const recoveredCount of [3, 1, 4]) {
test.provider(
`S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
(stack) =>
Effect.gen(function* () {
yield* stack.destroy();
const site = yield* stack.deploy(Fly.App("Site"));
try {
const proxy = yield* readinessProxy();
const initial = yield* readinessActor(
stack,
`S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
file,
proxy,
);
const first = yield* checkedDeployment(initial, proxy, "one", mode, 3, [0]);
yield* idleAll(site.appName, first.machineIds, mode);
const source = (yield* census(site.appName))[0]!;
const metadata = source.config!.metadata!;
const attemptBegin = proxy.events.length;
yield* Effect.gen(function* () {
try {
const gate = yield* Deferred.make<string>();
const actor = yield* readinessActor(
stack,
`S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
file,
proxy,
(event) =>
event.method === "POST" &&
event.path ===
`/v1/apps/${site.appName}/machines/${event.machineId}` &&
event.metadata?.["alchemy.replica"] === "1" &&
event.metadata["alchemy.fqn"] === metadata["alchemy.fqn"] &&
event.metadata["alchemy.generation"] !==
metadata["alchemy.generation"] &&
event.metadata["alchemy.readiness-roles"] === "run,run,idle" &&
event.metadata["alchemy.idle-policy-restored"] === "true" &&
event.phase === "promoting"
? Deferred.succeed(gate, event.machineId!).pipe(
Effect.andThen(Effect.never),
)
: Effect.void,
);
const attempt = yield* deployWorker(
actor,
"two",
topology(mode, 3, 2),
).pipe(Effect.scoped, Effect.forkScoped);
const barrierMachine = yield* Deferred.await(gate).pipe(
Effect.timeout("180 seconds"),
);
const pending = yield* census(site.appName);
expect(
pending.filter((machine) => first.machineIds.includes(machine.id!)),
).toHaveLength(3);
expect(
proxy.events.some((event) => retires(event, first.machineIds)),
).toBe(false);
const candidates = yield* Effect.forEach(
pending.filter((machine) => !first.machineIds.includes(machine.id!)),
(machine) =>
machines.getMachine({
app_name: site.appName,
machine_id: machine.id!,
}),
);
expect(candidates).toHaveLength(3);
expect(
candidates.every(
(machine) =>
machine.config?.metadata?.["alchemy.phase"] === "promoting",
),
).toBe(true);
const pendingSlots = assertSlots(candidates, [0, 1]);
expect(
pendingSlots[0]!.config?.metadata?.["alchemy.idle-policy-restored"],
).toBe("true");
expect(
pendingSlots[1]!.config?.metadata?.["alchemy.idle-policy-restored"],
).toBe("false");
expect(
proxy.readiness.some(
(event) =>
event.stage === "forwarded" &&
event.method === "GET" &&
event.machineId === pendingSlots[0]!.id &&
event.instanceId === pendingSlots[0]!.instance_id &&
event.metadata?.["alchemy.idle-policy-restored"] === "true" &&
event.phase === "promoting" &&
event.state === "started" &&
readinessChecksPassing(event.checks, [
"ready",
"servicecheck-00-http-80",
]),
),
).toBe(true);
expect(barrierMachine).toBe(pendingSlots[1]!.id);
expect(
autostopMode(pendingSlots[0]!.config?.services?.[0]?.autostop),
).toBe(mode);
expect(
autostopMode(pendingSlots[1]!.config?.services?.[0]?.autostop),
).toBe("off");
expect(
proxy.events
.slice(attemptBegin)
.some(
(event) =>
event.method === "POST" &&
event.path.endsWith(`/machines/${barrierMachine}`) &&
event.phase === "promoting",
),
).toBe(false);
const read = yield* observeReplicaSet({
appName: site.appName,
id: "Worker",
type: "Fly.Machine",
fqn: metadata["alchemy.fqn"]!,
resourceInstanceId: metadata["alchemy.instance"]!,
baseName: first.baseName,
machineIds: first.machineIds,
});
expect(read?.machineIds).toEqual(first.machineIds);
expect(read?.count).toBe(3);
expect(read?.rolloutPending).toBe(true);
const interruption = yield* Fiber.interrupt(attempt).pipe(
Effect.forkScoped,
);
yield* Effect.yieldNow;
yield* Effect.sync(() => {
proxy.clear();
proxy.release();
});
yield* Fiber.join(interruption).pipe(Effect.timeout("180 seconds"));
expect(Exit.hasInterrupts(yield* Fiber.await(attempt))).toBe(true);
expect(
proxy.events
.slice(attemptBegin)
.some(
(event) =>
first.machineIds.includes(event.machineId!) &&
event.path.endsWith("/start"),
),
).toBe(false);
expect(
proxy.readiness.some(
(event) =>
event.stage === "request" &&
event.path.endsWith("/metadata") &&
event.phase === "active" &&
candidates.some((machine) => machine.id === event.machineId),
),
).toBe(false);
expect(
proxy.events.some((event) => retires(event, first.machineIds)),
).toBe(false);
const resumed = yield* readinessActor(
stack,
`S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
file,
proxy,
);
const begin = proxy.readiness.length;
const recovered = yield* deployWorker(
resumed,
"two",
topology(mode, recoveredCount, 2),
).pipe(Effect.scoped);
expect(recovered.machineIds).toHaveLength(recoveredCount);
if (recoveredCount === 3) {
expect([...recovered.machineIds].sort()).toEqual(
candidates.map((machine) => machine.id).sort(),
);
} else {
expect(
recovered.machineIds.every(
(id) => !pending.some((machine) => machine.id === id),
),
).toBe(true);
}
const runningSlots = recoveredCount === 1 ? [0] : [0, 1];
const live = yield* assertTopology(
site.appName,
recovered.machineIds,
mode,
runningSlots,
2,
);
assertReadinessCommit(
proxy.readiness.slice(begin),
pending
.map((machine) => machine.id!)
.filter((id) => !recovered.machineIds.includes(id)),
live,
runningSlots,
["ready", "servicecheck-00-http-80"],
true,
);
const unchanged = yield* deployWorker(
resumed,
"two",
topology(mode, recoveredCount, 2),
).pipe(Effect.scoped);
expect(unchanged.machineIds).toEqual(recovered.machineIds);
const committed = yield* observeReplicaSet({
appName: site.appName,
id: "Worker",
type: "Fly.Machine",
fqn: metadata["alchemy.fqn"]!,
resourceInstanceId: metadata["alchemy.instance"]!,
machineIds: recovered.machineIds,
});
expect(committed?.rolloutPending).toBe(false);
expect(committed?.machineIds).toEqual(recovered.machineIds);
} finally {
yield* Effect.sync(() => {
proxy.clear();
proxy.release();
});
}
}).pipe(Effect.scoped);
} finally {
yield* stack.destroy();
yield* assertAppGone(site.appName);
}
}).pipe(Effect.scoped),
{ timeout: 900_000 },
);
}
test.provider(
`S11 public traffic autostarts ${mode} and Fly automatically returns it to idle within 15 minutes`,
(stack) =>
Effect.gen(function* () {
yield* stack.destroy();
const site = yield* stack.deploy(Fly.App("Site"));
try {
const output = yield* stack.deploy(
Effect.gen(function* () {
const app = yield* Fly.App("Site");
yield* Fly.IpAssignment("Public", { app, type: "shared_v4" });
return yield* Fly.Machine("Worker", {
app,
image: "nginx:alpine",
deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" },
shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
services: [
{
protocol: "tcp",
internalPort: 80,
autostop: mode,
autostart: true,
minMachinesRunning: 0,
ports: [{ port: 80, handlers: ["http"] }],
checks: [checks.ready],
},
],
});
}),
);
yield* idleAll(site.appName, output.machineIds, mode);
const client = yield* HttpClient.HttpClient;
yield* Effect.gen(function* () {
const response = yield* client.get(`http://${site.appName}.fly.dev`);
const body = yield* response.text;
if (response.status !== 200 || !body.includes("Welcome to nginx")) {
return yield* Effect.fail(
new Error(`Public Fly route is not ready: ${response.status}`),
);
}
expect(response.status).toBe(200);
expect(body).toContain("Welcome to nginx");
}).pipe(
Effect.retry({ schedule: Schedule.spaced("3 seconds"), times: 20 }),
Effect.timeout("90 seconds"),
);
yield* waitState(site.appName, output.machineId, "started");
const started = yield* Clock.currentTimeMillis;
const idleState = mode === "stop" ? "stopped" : "suspended";
// Only native GETs follow the last public request; no stop, suspend, or update substitutes.
const idle = yield* machines
.getMachine({ app_name: site.appName, machine_id: output.machineId })
.pipe(
Effect.tap((machine) =>
Effect.logInfo("Automatic Fly idle observation", {
mode,
state: machine.state,
instanceId: machine.instance_id,
}),
),
Effect.repeat({
schedule: Schedule.spaced("10 seconds"),
times: 90,
until: (machine) => machine.state === idleState,
}),
Effect.timeout("15 minutes"),
);
expect(idle.state).toBe(idleState);
expect(idle.cordoned).toBe(false);
expect(autostopMode(idle.config?.services?.[0]?.autostop)).toBe(mode);
expect(idle.config?.metadata?.["alchemy.idle-policy-restored"]).toBe("true");
yield* Effect.logInfo("Automatic return to idle observed", {
mode,
elapsedMs: (yield* Clock.currentTimeMillis) - started,
});
expect((yield* census(site.appName)).map((machine) => machine.id)).toEqual(
output.machineIds,
);
} finally {
yield* stack.destroy();
yield* assertAppGone(site.appName);
}
}),
{ tags: ["provider:fly:ipassignment"], timeout: 1_200_000 },
);
}
});
},
);
describe.sequential(
"idle capacity",
{
tags: [
"provider:fly",
"provider:fly:app",
"provider:fly:machine",
"provider:fly:service",
"live",
],
},
() => {
let endpoint: string | undefined;
const { test } = Test.make({ providers: throughProxy(() => endpoint) });
describe.sequential("idle topology", () => {
for (const mode of ["stop", "suspend"] as const) {
for (const allIdle of [false, true]) {
test.provider(
`S05 S11 ${mode} ${allIdle ? "all-idle" : "mixed"} replacement keeps an idle nonrepresentative`,
(stack) =>
Effect.gen(function* () {
yield* stack.destroy();
const props = {
count: 2,
services: [
{
protocol: "tcp",
internalPort: 80,
autostop: mode,
autostart: true,
minMachinesRunning: 0,
ports: [{ port: 80, handlers: ["http"] }],
checks: [checks.ready],
},
],
};
const initial = yield* deployWorker(stack, "one", {
...props,
deploy: { strategy: "rolling" },
});
for (const id of allIdle ? initial.machineIds : initial.machineIds.slice(1)) {
const target = { app_name: initial.appName, machine_id: id };
yield* mode === "stop"
? machines.stopMachine({ ...target, signal: "SIGQUIT", timeout: "5s" })
: machines.suspendMachine(target);
const observed = yield* machines.getMachine(target).pipe(
Effect.repeat({
schedule: Schedule.spaced("2 seconds"),
times: 8,
until: (machine) =>
machine.state === (mode === "stop" ? "stopped" : "suspended"),
}),
);
expect(observed.state).toBe(mode === "stop" ? "stopped" : "suspended");
}
const proxy = yield* transportProxy();
yield* Effect.sync(() => {
endpoint = proxy.url;
});
const next = yield* deployWorker(stack, "two", props);
expect(next.machineIds).toHaveLength(2);
const live = yield* census(initial.appName);
expect(live.map((machine) => machine.id).sort()).toEqual(
[...next.machineIds].sort(),
);
expect(
live.every((machine) => {
const autostop = machine.config?.services?.[0]?.autostop;
return (autostop === true ? "stop" : autostop) === mode;
}),
).toBe(true);
expect(
live.filter((machine) => machine.state === "started").length,
).toBeLessThanOrEqual(1);
expect(
live.some((machine) => ["stopped", "suspended"].includes(machine.state!)),
).toBe(true);
expect(
proxy.events.some(
(event) =>
initial.machineIds.includes(event.machineId!) &&
event.path.endsWith("/start"),
),
).toBe(false);
const same = yield* deployWorker(stack, "two", props);
expect(same.machineIds).toEqual(next.machineIds);
yield* Effect.sync(() => {
endpoint = undefined;
});
yield* stack.destroy();
yield* assertAppGone(initial.appName);
}).pipe(
Effect.scoped,
Effect.ensuring(
Effect.sync(() => {
endpoint = undefined;
}),
),
),
{ timeout: 300_000 },
);
}
}
});
},
);
describe.sequential(
"post-promotion health",
{
tags: [
"provider:fly",
"provider:fly:app",
"provider:fly:ipassignment",
"provider:fly:machine",
"provider:fly:secret",
"provider:fly:service",
"live",
],
},
() => {
let endpoint: string | undefined;
const { test } = Test.make({ providers: throughProxy(() => endpoint) });
describe.sequential("post-promotion health", () => {
for (const changed of [false, true]) {
test.provider(
`F09 a real post-uncordon health flip preserves both generations then recovers ${changed ? "changed" : "same"} desired state`,
(stack) =>
Effect.gen(function* () {
yield* stack.destroy();
const readiness = yield* makeReadinessControl();
const initial = yield* readiness.deployWorker(stack, "one");
const proxy = yield* transportProxy();
yield* Effect.sync(() => {
endpoint = proxy.url;
proxy.arm({
match: (event) => event.path.endsWith("/uncordon"),
action: "hold-response",
remaining: 1,
});
});
const update = yield* readiness
.deployWorker(stack, "two")
.pipe(Effect.result, Effect.forkScoped);
const promoted = yield* proxy.wait(
(event) =>
event.stage === "held" &&
event.path.endsWith("/uncordon") &&
event.status! < 300,
);
expect(promoted.machineId).toBeDefined();
yield* readiness.turnOff(initial.appName, promoted.machineId!);
yield* Effect.sync(proxy.release);
const result = yield* Fiber.join(update).pipe(Effect.timeout("60 seconds"));
expect(Result.isFailure(result)).toBe(true);
if (Result.isFailure(result))
expect(result.failure).toMatchObject({ _tag: "Fly.ReplicaChecksNotPassing" });
const live = yield* census(initial.appName);
expect(live.map((machine) => machine.id).sort()).toEqual(
[initial.machineId, promoted.machineId!].sort(),
);
expect(live.every((machine) => machine.cordoned === false)).toBe(true);
const pending = live.find((machine) => machine.id === promoted.machineId)!;
const metadata = pending.config!.metadata!;
expect(metadata["alchemy.phase"]).toBe("validating");
expect(metadata["alchemy.checked-instance"]).toBeUndefined();
const read = yield* observeReplicaSet({
appName: initial.appName,
id: "Worker",
type: "Fly.Machine",
fqn: metadata["alchemy.fqn"]!,
resourceInstanceId: metadata["alchemy.instance"]!,
machineIds: initial.machineIds,
});
expect(read?.machineIds).toEqual(initial.machineIds);
expect(read?.rolloutPending).toBe(true);
expect(
proxy.events.some(
(event) => event.method === "DELETE" && /\/machines\/[^/]+$/.test(event.path),
),
).toBe(false);
if (!changed) {
yield* repairReadiness(initial.appName, promoted.machineId!);
}
const recovered = yield* readiness.deployWorker(stack, changed ? "three" : "two");
if (changed) expect(recovered.machineId).not.toBe(promoted.machineId);
else expect(recovered.machineId).toBe(promoted.machineId);
yield* assertCommitted(initial.appName, recovered.machineIds);
yield* Effect.sync(() => {
endpoint = undefined;
});
yield* stack.destroy();
yield* assertAppGone(initial.appName);
}).pipe(
Effect.scoped,
Effect.ensuring(
Effect.sync(() => {
endpoint = undefined;
}),
),
),
{ timeout: 300_000 },
);
}
});
},
);
describe.sequential(
"ownership",
{
tags: [
"provider:fly",
"provider:fly:app",
"provider:fly:machine",
"provider:fly:service",
"live",
],
},
() => {
const { test } = Test.make({ providers: Fly.providers() });
test.provider(
"S09 foreign routed Machine and sibling survive rollout and stale-instance deletion",
(stack) =>
Effect.gen(function* () {
yield* stack.destroy();
const deploy = (version: string) =>
stack.deploy(
Effect.gen(function* () {
const app = yield* Fly.App("Site");
const sibling = yield* Fly.Machine("Sibling", {
app,
image: "nginx:alpine",
env: { VERSION: "sibling" },
});
const worker = yield* Fly.Machine("Worker", {
app,
image: "nginx:alpine",
env: { VERSION: version },
checks,
deploy: { strategy: "bluegreen", healthTimeout: "20 seconds" },
shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
});
return { sibling, worker };
}),
);
const initial = yield* deploy("one");
const foreign = yield* machines.createMachine({
app_name: initial.worker.appName,
name: "foreign-routed",
region: "iad",
config: {
image: "nginx:alpine",
env: { VERSION: "foreign" },
services: [
{
protocol: "tcp",
internal_port: 80,
ports: [{ port: 80, handlers: ["http"] }],
},
],
},
});
const removeForeign = machines
.deleteMachine({
app_name: initial.worker.appName,
machine_id: foreign.id!,
force: true,
})
.pipe(Effect.catchTag("NotFound", () => Effect.void));
yield* Effect.gen(function* () {
const next = yield* deploy("two");
expect(next.worker.machineId).not.toBe(initial.worker.machineId);
expect(next.sibling.machineId).toBe(initial.sibling.machineId);
const before = yield* census(initial.worker.appName);
expect(before.map((machine) => machine.id).sort()).toEqual(
[next.worker.machineId, next.sibling.machineId, foreign.id!].sort(),
);
expect(before.find((machine) => machine.id === foreign.id)?.cordoned).toBe(false);
const owned = before.find((machine) => machine.id === next.worker.machineId)!;
yield* deleteReplicaSet({
appName: next.worker.appName,
id: "Worker",
type: "Fly.Machine",
fqn: owned.config!.metadata!["alchemy.fqn"]!,
resourceInstanceId: "stale-instance-must-not-delete-successor",
machineIds: next.worker.machineIds,
volumeIds: [],
});
expect(
(yield* census(initial.worker.appName)).map((machine) => machine.id).sort(),
).toEqual(before.map((machine) => machine.id).sort());
}).pipe(Effect.ensuring(removeForeign.pipe(Effect.orDie)));
yield* stack.destroy();
yield* assertAppGone(initial.worker.appName);
}),
{ timeout: 300_000 },
);
test.provider(
"S09 F11 stale output and a same-logical-ID foreign FQN cannot authorize successor deletion",
(stack) =>
Effect.gen(function* () {
yield* stack.destroy();
const deploy = (name: string, version: string) =>
stack.deploy(
Effect.gen(function* () {
const app = yield* Fly.App("Site");
return yield* Fly.Machine("Worker", {
app,
name,
image: "nginx:alpine",
checks,
env: { VERSION: version },
deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
});
}),
);
const initial = yield* deploy("stale-original", "one");
const old = yield* machines.getMachine({
app_name: initial.appName,
machine_id: initial.machineId,
});
const oldMetadata = old.config!.metadata!;
const collision = yield* machines.createMachine({
app_name: initial.appName,
name: "fqn-collision",
region: "iad",
config: {
...old.config,
metadata: {
...oldMetadata,
"alchemy.fqn": `${oldMetadata["alchemy.fqn"]}/OtherScope`,
},
},
});
yield* Effect.gen(function* () {
const next = yield* deploy("stale-original", "two");
const refreshed = yield* observeReplicaSet({
appName: initial.appName,
id: "Worker",
type: "Fly.Machine",
fqn: oldMetadata["alchemy.fqn"]!,
resourceInstanceId: oldMetadata["alchemy.instance"]!,
baseName: initial.baseName,
machineIds: [...initial.machineIds, collision.id!],
});
expect(refreshed?.machineIds).toEqual(next.machineIds);
expect(refreshed?.rolloutPending).toBe(false);
const successor = yield* deploy("stale-successor", "three");
const current = yield* machines.getMachine({
app_name: initial.appName,
machine_id: successor.machineId,
});
expect(current.config?.metadata?.["alchemy.instance"]).not.toBe(
oldMetadata["alchemy.instance"],
);
// Even an ID cache containing the new Machine grants no old-lineage authority.
yield* deleteReplicaSet({
appName: initial.appName,
id: "Worker",
type: "Fly.Machine",
fqn: oldMetadata["alchemy.fqn"]!,
resourceInstanceId: oldMetadata["alchemy.instance"]!,
machineIds: [
...initial.machineIds,
...next.machineIds,
...successor.machineIds,
collision.id!,
],
volumeIds: [],
});
const live = yield* census(initial.appName);
expect(live.map((machine) => machine.id).sort()).toEqual(
[successor.machineId, collision.id!].sort(),
);
const foreign = live.find((machine) => machine.id === collision.id)!;
expect(foreign.config?.metadata?.["alchemy.fqn"]).toBe(
`${oldMetadata["alchemy.fqn"]}/OtherScope`,
);
expect(foreign.cordoned).toBe(false);
}).pipe(
Effect.ensuring(
machines
.deleteMachine({
app_name: initial.appName,
machine_id: collision.id!,
force: true,
})
.pipe(
Effect.catchTag("NotFound", () => Effect.void),
Effect.orDie,
),
),
);
yield* stack.destroy();
yield* assertAppGone(initial.appName);
}),
{ timeout: 600_000 },
);
},
);
describe.sequential(
"deployment",
{
tags: [
"provider:fly",
"provider:fly:app",
"provider:fly:machine",
"provider:fly:service",
"live",
],
},
() => {
const { test } = Test.make({ providers: Fly.providers() });
test.provider(
"S01 bluegreen worker checks, promotion, replacement, and graceful teardown",
(stack) =>
Effect.gen(function* () {
yield* stack.destroy();
const deploy = (version: string, path = "/") =>
stack.deploy(
Effect.gen(function* () {
const app = yield* Fly.App("Site");
return yield* Fly.Machine("Worker", {
app,
name: "bluegreen-worker-long-name-base",
image: "nginx:alpine",
env: { VERSION: version },
deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
shutdown: { signal: "SIGQUIT", timeout: "10 seconds" },
checks: {
ready: { type: "http", port: 80, path, interval: "2s", timeout: "1s" },
},
});
}),
);
const first = yield* deploy("one");
const committed = yield* machines.getMachine({
app_name: first.appName,
machine_id: first.machineId,
});
// Active commit metadata can reset reports after the provider's readiness validation.
const machine = yield* waitHealthy(first.appName, committed, 30_000);
expect(machine.instance_id).toBe(committed.instance_id);
yield* Effect.logInfo("Observed promoted worker", {
checks: machine.checks?.map(({ name, status }) => ({ name, status })),
configured: machine.config?.checks,
metadata: machine.config?.metadata,
});
expect(machine.cordoned).toBe(false);
expect(
machine.checks?.some(
(check) => check.name === "ready" && check.status === "passing",
),
).toBe(true);
expect(machine.config?.metadata?.["alchemy.phase"]).toBe("active");
expect(first.name.length).toBeLessThanOrEqual(30);
const second = yield* deploy("two");
expect(second.machineId).not.toBe(first.machineId);
expect(second.baseName).toBe(first.baseName);
const listed = yield* machines.listMachines({ app_name: first.appName });
expect(
listed
.filter((machine) => machine.state !== "destroyed")
.map((machine) => machine.id),
).toEqual([second.machineId]);
yield* stack.destroy();
yield* assertAppGone(first.appName);
}),
{ timeout: 180_000 },
);
test.provider(
"S04 unhealthy replacement preserves the old routed ID and cleans unpromoted candidates",
(stack) =>
Effect.gen(function* () {
yield* stack.destroy();
const initial = yield* deployWorker(stack, "one");
const failed = yield* deployWorker(stack, "broken", {
checks: { ready: { ...checks.ready, path: "/missing" } },
deploy: { strategy: "bluegreen", healthTimeout: "8 seconds" },
}).pipe(Effect.result);
expect(Result.isFailure(failed)).toBe(true);
if (Result.isFailure(failed))
expect(failed.failure).toMatchObject({ _tag: "Fly.ReplicaChecksNotPassing" });
const live = yield* census(initial.appName);
expect(live.map((machine) => machine.id)).toEqual(initial.machineIds);
expect(live[0]!.state).toBe("started");
expect(live[0]!.cordoned).toBe(false);
yield* stack.destroy();
yield* assertAppGone(initial.appName);
}),
{ timeout: 180_000 },
);
},
);
describe.sequential(
"promotion faults",
{
tags: [
"provider:fly",
"provider:fly:app",
"provider:fly:machine",
"provider:fly:service",
"live",
],
},
() => {
let endpoint: string | undefined;
const { test } = Test.make({ providers: throughProxy(() => endpoint) });
test.provider(
"F03 F08 partial promotion loses the first uncordon response without rolling back serving green",
(stack) =>
Effect.gen(function* () {
yield* stack.destroy();
const initial = yield* deployWorker(stack, "one", { count: 2 });
const proxy = yield* transportProxy();
const match = (event: { path: string }) => event.path.endsWith("/uncordon");
yield* Effect.sync(() => {
endpoint = proxy.url;
proxy.arm({ match, action: "drop-response", remaining: 1 });
proxy.arm({ match, action: "cut-request", remaining: Infinity });
});
try {
const failed = yield* deployWorker(stack, "two", { count: 2 }).pipe(
Effect.timeout("120 seconds"),
Effect.result,
);
expect(Result.isFailure(failed)).toBe(true);
const lost = proxy.events.find(
(event) => event.stage === "dropped" && event.path.endsWith("/uncordon"),
);
expect(lost?.status).toBeGreaterThanOrEqual(200);
expect(lost?.status).toBeLessThan(300);
const live = yield* census(initial.appName);
expect(
initial.machineIds.every((id) =>
live.some((machine) => machine.id === id && machine.cordoned === false),
),
).toBe(true);
expect(live.find((machine) => machine.id === lost!.machineId)?.cordoned).toBe(
false,
);
const candidateIds = live
.filter((machine) => !initial.machineIds.includes(machine.id!))
.map((machine) => machine.id!);
expect(candidateIds).toHaveLength(2);
expect(
proxy.events.some(
(event) =>
event.method === "DELETE" &&
/\/machines\/[^/]+$/.test(event.path) &&
candidateIds.includes(event.machineId!),
),
).toBe(false);
yield* Effect.sync(proxy.clear);
const recovered = yield* deployWorker(stack, "two", { count: 2 });
expect([...recovered.machineIds].sort()).toEqual(candidateIds.sort());
yield* assertCommitted(initial.appName, recovered.machineIds);
} finally {
yield* Effect.sync(() => {
endpoint = undefined;
proxy.clear();
});
}
yield* stack.destroy();
yield* assertAppGone(initial.appName);
}).pipe(
Effect.scoped,
Effect.ensuring(
Effect.sync(() => {
endpoint = undefined;
}),
),
),
{ timeout: 300_000 },
);
},
);
describe.sequential(
"transport",
{
tags: [
"provider:fly",
"provider:fly:app",
"provider:fly:machine",
"provider:fly:service",
"live",
],
},
() => {
let endpoint: string | undefined;
const { test } = Test.make({ providers: throughProxy(() => endpoint) });
test.provider(
"F02 S07 harness forwards real Fly responses, loses completed responses and cuts connections",
(stack) =>
Effect.gen(function* () {
yield* stack.destroy();
const app = yield* stack.deploy(Fly.App("Site"));
const proxy = yield* transportProxy();
yield* Effect.sync(() => {
endpoint = proxy.url;
});
try {
const request = machines.getApp({ app_name: app.appName }).pipe(Retry.none);
const forwarded = yield* request;
expect(forwarded.name).toBe(app.appName);
expect(
proxy.events.some((event) => event.stage === "forwarded" && event.status === 200),
).toBe(true);
yield* Effect.sync(() =>
proxy.arm({
match: (event) => event.path.endsWith(`/apps/${app.appName}`),
action: "drop-response",
remaining: 1,
}),
);
const dropped = yield* request.pipe(Effect.result);
// The HTTP runtime may transparently replay an idempotent GET after a socket reset.
if (Result.isFailure(dropped)) expect(dropped.failure._tag).toBe("HttpClientError");
else expect(dropped.success.name).toBe(app.appName);
expect(
proxy.events.filter((event) => event.stage === "dropped" && event.status === 200),
).toHaveLength(1);
yield* Effect.sync(() =>
proxy.arm({ match: () => true, action: "cut-request", remaining: Infinity }),
);
const cut = yield* request.pipe(Effect.result);
expect(Result.isFailure(cut)).toBe(true);
const cutEvent = proxy.events.find((event) => event.stage === "cut")!;
expect(
proxy.events.some(
(event) => event.sequence === cutEvent.sequence && event.stage === "completed",
),
).toBe(false);
yield* Effect.sync(() => {
proxy.clear();
proxy.arm({ match: () => true, action: "hold-response", remaining: 1 });
});
const child = yield* request.pipe(Effect.forkScoped);
const held = yield* proxy.wait((event) => event.stage === "held");
expect(held.status).toBe(200);