import type {
FlyMachineConfig,
FlyMachineMount,
FlyMachineService,
FlyMachineServiceCheck,
FlyStopConfig,
ImageRef as FlyImageRef,
Machine as FlyMachine,
Volume as FlyVolume,
} from "@distilled.cloud/fly-io/machines";
import * as machines from "@distilled.cloud/fly-io/machines";
import * as Retry from "@distilled.cloud/fly-io/Retry";
import { deepEqual } from "alchemy/Diff";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Result from "effect/Result";
import * as Schedule from "effect/Schedule";
import { listOwnedApps } from "./App.ts";
import { reconcileBlueGreen, setRouting } from "./bluegreen.ts";
import { validateDeployment, type DeploymentPolicy, type MachineCheck } from "./Deployment.ts";
import { classifyDeploymentState, validProtocol2Generation } from "./DeploymentState.ts";
import { usingMachineLeases, type MachineLeases } from "./leases.ts";
import type {
MachineGuest,
MachineImageRef,
MachineService,
MachineServiceCheck,
} from "./Machine.ts";
import {
alchemyMetadataKeys,
createMachineMetadata,
isAlchemyOwnedMetadata,
sanitizeFlyAppName,
type FlyAlchemyType,
} from "./Metadata.ts";
import type { DiskSpec, MountedDisk } from "./MountVolume.ts";
import { regionOfReplica } from "./Region.ts";
import { deleteVolume, ensureVolumeGroup, getVolumeById, volumeGroupName } from "./Volume.ts";
const WAIT_TIMEOUT_SECONDS = 8;
const waitBackoff = Schedule.exponential("500 millis");
const SERVICE_CHECK_NAME_PREFIX = "servicecheck-";
export class ReplicaNotCreated extends Data.TaggedError("Fly.ReplicaNotCreated")<{
resourceName: string;
appName: string;
}> {}
export class ReplicaChecksNotPassing extends Data.TaggedError("Fly.ReplicaChecksNotPassing")<{
appName: string;
machineId: string;
checks: ReadonlyArray<{
name: string | undefined;
status: string | undefined;
output: string | undefined;
}>;
}> {
get message() {
const checks = this.checks.map(
(check) =>
`${check.name ?? "unnamed"}: ${check.status ?? "unknown"}${check.output ? ` (${check.output})` : ""}`,
);
return `Service checks did not pass for ${this.appName}/${this.machineId}: ${checks.join("; ") || "no service check results"}`;
}
}
export interface Replica {
machineId: string;
name: string;
baseName?: string;
region: string;
state: string;
instanceId: string | undefined;
privateIp: string | undefined;
imageRef: MachineImageRef | undefined;
guest: MachineGuest | undefined;
mounts: MountedDisk[];
}
export interface ReplicaSet {
rolloutPending?: boolean;
appName: string;
machineId: string;
machineIds: string[];
name: string;
baseName?: string;
/** Region of replica 0. */
region: string;
/** Regions the replicas run in, in replica order. */
regions: string[];
state: string;
instanceId: string | undefined;
privateIp: string | undefined;
imageRef: MachineImageRef | undefined;
guest: MachineGuest | undefined;
/** Observed proxy services published by replica 0. */
services: FlyMachineService[] | undefined;
count: number;
mounts: MountedDisk[];
replicas: Replica[];
}
const compactRecord = (
record: Record<string, string | undefined> | null | undefined,
): Record<string, string> =>
Object.fromEntries(
Object.entries(record ?? {}).flatMap(([key, value]) =>
value === undefined ? [] : [[key, value]],
),
);
export const gone = (machine: FlyMachine | undefined) =>
machine === undefined || machine.state === "destroyed";
export const getMachineById = (appName: string, machineId: string) =>
machines.getMachine({ app_name: appName, machine_id: machineId }).pipe(
Effect.map((machine) => (gone(machine) ? undefined : machine)),
Effect.catchTag("NotFound", () => Effect.undefined),
);
/**
* Reads a Machine we hold a lease on. Fly's Machines API is not
* read-after-write consistent: a freshly created Machine can accept a lease
* (201) and then answer `GET /machines/{id}` with 404 for several seconds.
* Holding the lease blocks other writers, so a 404 here is lag rather than
* deletion; retry it (bounded) before surfacing `NotFound`.
*/
const getLeasedMachine = (appName: string, machineId: string) =>
machines.getMachine({ app_name: appName, machine_id: machineId }).pipe(
Effect.retry({
times: 10,
schedule: Schedule.spaced("1 second"),
while: (error) => error._tag === "NotFound",
}),
);
const getLeasedMachineById = (appName: string, machineId: string) =>
getLeasedMachine(appName, machineId).pipe(
Effect.map((machine) => (gone(machine) ? undefined : machine)),
Effect.catchTag("NotFound", () => Effect.undefined),
);
export const listMachinesByApp = (appName: string) =>
machines.listMachines({ app_name: appName }).pipe(
// A just-created App can answer 500 "failed to get app: sql: no rows in
// result set" until the Machines API sees it; safe to replay a read.
Effect.retry({
while: (error) => error._tag === "InternalServerError",
schedule: Schedule.spaced("2 seconds"),
times: 8,
}),
Effect.map((machines) => machines.filter((machine) => !gone(machine))),
Effect.catchTag("NotFound", () => Effect.succeed([])),
);
export const resolveCount = (count: number | undefined) => Math.max(1, Math.floor(count ?? 1));
export const replicaMachineName = (base: string, index: number, count: number) => {
if (count <= 1 && index === 0) return base;
const suffix = `-${index}`;
const room = 30 - suffix.length;
const clipped = base.slice(0, Math.max(1, room)).replace(/-+$/g, "");
return sanitizeFlyAppName(`${clipped}${suffix}`);
};
export const replicaIndexOf = (machine: FlyMachine): number => {
const raw = compactRecord(machine.config?.metadata)[alchemyMetadataKeys.replica];
const parsed = raw === undefined ? 0 : Number(raw);
return Number.isFinite(parsed) ? parsed : 0;
};
export const alchemyIdOf = (machine: FlyMachine): string | undefined => {
const id = compactRecord(machine.config?.metadata)[alchemyMetadataKeys.id];
return id !== undefined && id.length > 0 ? id : undefined;
};
export const isOwnedType = (machine: FlyMachine, type: FlyAlchemyType) => {
const metadata = compactRecord(machine.config?.metadata);
return isAlchemyOwnedMetadata(metadata) && metadata[alchemyMetadataKeys.type] === type;
};
export const toImageRef = (ref: FlyImageRef | undefined): MachineImageRef | undefined => {
if (ref === undefined) return undefined;
const imageRef: MachineImageRef = {
registry: ref.registry,
repository: ref.repository,
tag: ref.tag,
digest: ref.digest,
};
return imageRef.registry === undefined &&
imageRef.repository === undefined &&
imageRef.tag === undefined &&
imageRef.digest === undefined
? undefined
: imageRef;
};
export const toGuestAttrs = (
guest:
| { cpu_kind?: string; cpus?: number; memory_mb?: number; gpu_kind?: string; gpus?: number }
| undefined,
): MachineGuest | undefined => {
if (guest === undefined) return undefined;
return {
cpuKind: guest.cpu_kind,
cpus: guest.cpus,
memoryMb: guest.memory_mb,
gpuKind: guest.gpu_kind,
gpus: guest.gpus,
};
};
export const toFlyServiceCheck = (check: MachineServiceCheck): FlyMachineServiceCheck => ({
type: check.type,
port: check.port,
interval: check.interval,
timeout: check.timeout,
grace_period: check.gracePeriod,
method: check.method,
path: check.path,
protocol: check.protocol,
headers: check.headers?.map((header) => ({ name: header.name, values: header.values })),
tls_server_name: check.tlsServerName,
tls_skip_verify: check.tlsSkipVerify,
});
export const toFlyService = (service: MachineService): FlyMachineService => ({
protocol: service.protocol,
internal_port: service.internalPort,
autostart: service.autostart,
autostop:
typeof service.autostop === "boolean" ? (service.autostop ? "stop" : "off") : service.autostop,
min_machines_running: service.minMachinesRunning,
ports: service.ports?.map((port) => ({
port: port.port,
handlers: port.handlers,
force_https: port.forceHttps,
start_port: port.startPort,
end_port: port.endPort,
})),
checks: service.checks?.map((check) => ({
...toFlyServiceCheck(check),
// Fly clamps service-check intervals; named Machine checks retain longer intervals.
interval:
(durationNanoseconds(check.interval) ?? 0n) > 60_000_000_000n ? "60s" : check.interval,
})),
});
export const autostopMode = (value: string | boolean | undefined) =>
value === true ? "stop" : value === false || value === undefined ? "off" : value;
const normalizedCheckDuration = (value: string | undefined) => {
const nanos = durationNanoseconds(value);
return nanos === undefined ? value : `${nanos}ns`;
};
const normalizedCheck = (check: machines.FlyMachineCheck | FlyMachineServiceCheck | undefined) =>
check === undefined
? undefined
: {
...check,
grace_period: normalizedCheckDuration(check.grace_period),
interval: normalizedCheckDuration(check.interval),
timeout: normalizedCheckDuration(check.timeout),
};
export const sameChecks = (
observed: FlyMachineConfig["checks"],
desired: FlyMachineConfig["checks"],
) => {
const normalize = (checks: FlyMachineConfig["checks"]) =>
Object.fromEntries(
Object.entries(checks ?? {}).map(([name, check]) => [name, normalizedCheck(check)]),
);
return deepEqual(normalize(observed), normalize(desired), { stripNullish: true });
};
export const normalizedServices = (services: FlyMachineService[] | undefined) =>
(services ?? []).map((service) => ({
...service,
autostop: autostopMode(service.autostop),
checks: service.checks?.map(normalizedCheck),
}));
export const sameServices = (
observed: FlyMachineService[] | undefined,
desired: FlyMachineService[] | undefined,
) => deepEqual(normalizedServices(observed), normalizedServices(desired), { stripNullish: true });
export const hasPublishedService = (services: FlyMachineService[] | undefined) =>
(services ?? []).some((service) =>
(service.ports ?? []).some((port) => port.port !== undefined || port.start_port !== undefined),
);
export const waitStarted = (appName: string, machineId: string) =>
machines
.waitMachine({
app_name: appName,
machine_id: machineId,
state: "started",
timeout: WAIT_TIMEOUT_SECONDS,
})
.pipe(
Retry.none,
Effect.retry({
times: 6,
schedule: waitBackoff,
while: (e) => e._tag === "GatewayTimeout" || e._tag === "MachineWaitTimeout",
}),
Effect.timeout("50 seconds"),
);
export const configuredCheckNames = (config: FlyMachineConfig | undefined) => [
...Object.keys(config?.checks ?? {}),
...(config?.services ?? []).flatMap((service) =>
(service.checks ?? []).map(
(check, checkIndex) =>
`${SERVICE_CHECK_NAME_PREFIX}${String(checkIndex).padStart(2, "0")}-${check.type ?? "tcp"}-${service.internal_port}`,
),
),
];
export const checksPassing = (machine: FlyMachine, config: FlyMachineConfig | undefined) => {
const expected = configuredCheckNames(config);
const checks = machine.checks ?? [];
// Fly may also report compatibility mirrors; they do not replace service reports.
return (
machine.state === "started" &&
new Set(expected).size === expected.length &&
checks.every(
(check) =>
check.name !== undefined &&
check.status === "passing" &&
(expected.includes(check.name) ||
expected.some(
(name) =>
name.startsWith(SERVICE_CHECK_NAME_PREFIX) &&
check.name === name.replace(SERVICE_CHECK_NAME_PREFIX, "bg_deployments_compat-"),
)),
) &&
new Set(checks.map((check) => check.name)).size === checks.length &&
expected.every((name) => {
const reports = checks.filter((check) => check.name === name);
return reports.length === 1 && reports[0]?.status === "passing";
})
);
};
const TRANSIENT_GET_TAGS = [
"TooManyRequests",
"InternalServerError",
"BadGateway",
"ServiceUnavailable",
"GatewayTimeout",
] as const;
/**
* After the Machine is started, wait until Fly reports every service
* check as passing. No configured checks is a no-op. Empty live
* `servicecheck-*` results keep polling — they are not success.
* `warning` / `unknown` during grace keep polling.
*/
export const waitHealthy = Effect.fn(function* (
appName: string,
machine: FlyMachine,
healthTimeoutMs = 60_000,
config: FlyMachineConfig | undefined = machine.config,
) {
const machineId = machine.id;
const named = Object.keys(config?.checks ?? {});
const expected = (config?.services ?? []).flatMap((service) => service.checks ?? []).length;
if (machineId === undefined || (expected === 0 && named.length === 0)) return machine;
let observed = machine;
const notPassing = () =>
new ReplicaChecksNotPassing({
appName,
machineId,
checks: (observed.checks ?? []).map((check) => ({
name: check.name,
status: check.status,
output: undefined,
})),
});
const passing = yield* getMachineById(appName, machineId).pipe(
Retry.none,
Effect.map((current) => {
if (current === undefined) return false;
observed = current;
return (
current.instance_id !== undefined &&
current.instance_id === machine.instance_id &&
checksPassing(current, config)
);
}),
Effect.catchTag(TRANSIENT_GET_TAGS, () => Effect.succeed(false)),
Effect.catchReason("HttpClientError", "TransportError", () => Effect.succeed(false)),
Effect.repeat({
schedule: Schedule.spaced(healthTimeoutMs / 10),
until: (passing) => passing,
times: 10,
}),
Effect.timeoutOrElse({ duration: healthTimeoutMs, orElse: () => Effect.fail(notPassing()) }),
);
if (!passing) return yield* notPassing();
return observed;
});
export const waitDestroyed = (appName: string, machineId: string) =>
machines
.waitMachine({
app_name: appName,
machine_id: machineId,
state: "destroyed",
timeout: WAIT_TIMEOUT_SECONDS,
})
.pipe(
Retry.none,
Effect.as(undefined),
Effect.catchTag("NotFound", () => Effect.void),
Effect.retry({
times: 6,
schedule: waitBackoff,
while: (e) => e._tag === "GatewayTimeout" || e._tag === "MachineWaitTimeout",
}),
Effect.timeout("50 seconds"),
);
export const ensureStarted = (
appName: string,
machine: FlyMachine,
skipLaunch: boolean,
healthTimeoutMs = 60_000,
expectedConfig?: FlyMachineConfig,
existingLeases?: MachineLeases,
) =>
usingMachineLeases(appName, existingLeases, (leases) =>
ensureLeasedStarted(appName, machine, skipLaunch, healthTimeoutMs, expectedConfig, leases),
);
const ensureLeasedStarted = Effect.fn(function* (
appName: string,
machine: FlyMachine,
skipLaunch: boolean,
healthTimeoutMs: number,
expectedConfig: FlyMachineConfig | undefined,
leases: MachineLeases,
) {
const machineId = machine.id;
if (machineId === undefined || skipLaunch) return machine;
yield* leases.acquire([machineId]);
const started = yield* Effect.gen(function* () {
// Create/update responses can lag Fly's automatic launch.
const current = yield* getLeasedMachine(appName, machineId);
if (!sameOwnership(current, machine)) {
return yield* new ReplicaOwnershipChanged({ appName, machineId });
}
yield* Effect.logDebug("Fly machine startup", { appName, machineId, state: current.state });
if (
current.state === "stopped" ||
current.state === "suspended" ||
current.state === "failed"
) {
yield* leases.mutate(machineId, (lease_nonce) =>
machines.startMachine({ app_name: appName, machine_id: machineId, lease_nonce }),
);
}
// Re-observe state between waits instead of retrying the wait in the SDK.
yield* machines
.waitMachine({
app_name: appName,
machine_id: machineId,
state: "started",
instance_id: current.instance_id,
timeout: WAIT_TIMEOUT_SECONDS,
})
.pipe(Retry.none);
return yield* getLeasedMachine(appName, machineId);
}).pipe(
Effect.retry({
times: 6,
schedule: waitBackoff,
while: (error) =>
error._tag === "MachineStartFromCreatedState" ||
error._tag === "MachineReplacing" ||
error._tag === "MachineWaitTimeout" ||
error._tag === "Conflict" ||
error._tag === "GatewayTimeout",
}),
Effect.timeout("180 seconds"),
);
return yield* waitHealthy(appName, started, healthTimeoutMs, expectedConfig ?? machine.config);
});
export const deleteMachine = (appName: string, machineId: string, existingLeases?: MachineLeases) =>
usingMachineLeases(appName, existingLeases, (leases) =>
Effect.gen(function* () {
if (appName.length === 0 || machineId.length === 0) return;
yield* leases.acquire([machineId]);
yield* leases.remove(machineId, (lease_nonce) =>
machines
.deleteMachine({ app_name: appName, machine_id: machineId, force: true, lease_nonce })
.pipe(Effect.timeout("30 seconds")),
);
yield* waitDestroyed(appName, machineId);
}).pipe(Effect.catchTag("NotFound", () => leases.forget(machineId))),
);
const durationNanoseconds = (value: string | undefined) => {
if (value === undefined || value === "") return undefined;
const negative = value.startsWith("-");
const duration = /^[+-]/.test(value) ? value.slice(1) : value;
if (duration === "0") return 0n;
const limit = negative ? 1n << 63n : (1n << 63n) - 1n;
const units: Record<string, bigint> = {
ns: 1n,
us: 1_000n,
µs: 1_000n,
μs: 1_000n,
ms: 1_000_000n,
s: 1_000_000_000n,
m: 60_000_000_000n,
h: 3_600_000_000_000n,
};
let total = 0n;
let consumed = 0;
for (const part of duration.matchAll(/(\d+(?:\.\d*)?|\.\d+)(ns|us|µs|μs|ms|s|m|h)/gy)) {
const [integer = "", fraction = ""] = part[1]!.split(".");
const whole = integer.replace(/^0+/, "") || "0";
if (whole.length > 19) return undefined;
const unit = units[part[2]!]!;
let fractionalNanos = 0n;
// Truncate each component to nanoseconds without unbounded BigInt operands.
for (let index = fraction.length - 1; index >= 0; index--) {
fractionalNanos = (BigInt(fraction[index]!) * unit + fractionalNanos) / 10n;
}
total += BigInt(whole) * unit + fractionalNanos;
if (total > limit) return undefined;
consumed += part[0].length;
}
if (consumed === 0 || consumed !== duration.length) return undefined;
return negative ? -total : total;
};
const stopTimeoutMillis = (timeout: string | undefined) => {
if (timeout === "") return 0;
// Shutdown policies retain their unsigned millisecond-or-larger syntax.
if (timeout === undefined || !/^(?:\d+(?:\.\d+)?(?:ms|s|m|h))+$/.test(timeout)) {
return undefined;
}
const nanos = durationNanoseconds(timeout);
return nanos === undefined ? undefined : Number(nanos) / 1_000_000;
};
export const sameStopConfig = (
observed: FlyStopConfig | undefined,
desired: FlyStopConfig | undefined,
) =>
observed?.signal === desired?.signal &&
stopTimeoutMillis(observed?.timeout) === stopTimeoutMillis(desired?.timeout);
export class ShutdownPolicyMismatch extends Data.TaggedError("Fly.ShutdownPolicyMismatch")<{
machineId: string;
message: string;
}> {}
export const predecessorShutdown = Effect.fn(function* (machine: FlyMachine) {
const persisted = machine.config?.stop_config;
const injected = machine.config?.env?.ALCHEMY_FLY_SHUTDOWN_TIMEOUT_MS;
const timeoutMs = stopTimeoutMillis(persisted?.timeout);
if (injected !== undefined) {
const managedTimeout = Number(injected);
const signal = persisted?.signal ?? "SIGTERM";
if (
!/^\d+$/.test(injected) ||
!Number.isSafeInteger(managedTimeout) ||
managedTimeout <= 0 ||
managedTimeout > 300_000 ||
(signal !== "SIGTERM" && signal !== "SIGINT") ||
(persisted?.timeout !== undefined && timeoutMs !== managedTimeout)
) {
return yield* new ShutdownPolicyMismatch({
machineId: machine.id ?? "",
message:
"The predecessor's managed shutdown environment and stop_config disagree; reconcile its shutdown policy before migration.",
});
}
return {
signal,
timeout: persisted?.timeout ?? `${managedTimeout}ms`,
timeoutMs: managedTimeout,
};
}
if (persisted?.timeout !== undefined && (!timeoutMs || timeoutMs > 300_000)) {
return yield* new ShutdownPolicyMismatch({
machineId: machine.id ?? "",
message:
"The predecessor has an invalid stop_config timeout; refusing to shorten its shutdown window.",
});
}
// Unspecified raw-image overrides must remain unspecified for Fly's defaults.
return {
signal: persisted?.signal,
timeout: persisted?.timeout,
timeoutMs: timeoutMs ?? 300_000,
};
});
export class ReplicaOwnershipChanged extends Data.TaggedError("Fly.ReplicaOwnershipChanged")<{
appName: string;
machineId: string;
}> {}
export class ReplicaRetirementIncomplete extends Data.TaggedError(
"Fly.ReplicaRetirementIncomplete",
)<{ appName: string; residuals: Array<{ machineId: string; stage: string }> }> {}
const retirementStep =
(appName: string, machineId: string, stage: string) =>
<A, E extends { readonly _tag: string }, R>(effect: Effect.Effect<A, E, R>) =>
effect.pipe(
Effect.mapError((error) =>
error._tag === "NotFound"
? error
: new ReplicaRetirementIncomplete({
appName,
residuals: [{ machineId, stage: `${stage}: ${error._tag}` }],
}),
),
);
const sameOwnership = (observed: FlyMachine, snapshot: FlyMachine) =>
observed.id === snapshot.id &&
[
alchemyMetadataKeys.stack,
alchemyMetadataKeys.stage,
alchemyMetadataKeys.id,
alchemyMetadataKeys.type,
alchemyMetadataKeys.instance,
alchemyMetadataKeys.fqn,
alchemyMetadataKeys.generation,
].every((key) => observed.config?.metadata?.[key] === snapshot.config?.metadata?.[key]);
export const leaseSnapshot = Effect.fn(function* (
appName: string,
snapshot: FlyMachine[],
leases: MachineLeases,
) {
const ordered = [...snapshot].sort((a, b) => (a.id ?? "").localeCompare(b.id ?? ""));
const leased = new Set<string>();
yield* Effect.forEach(
ordered,
(machine) =>
Effect.gen(function* () {
if (!machine.id || machine.host_status === "unreachable") return;
const id = machine.id;
yield* leases.acquire([id]).pipe(
Effect.tap(() => Effect.sync(() => leased.add(id))),
// A just-created Machine can 404 on the lease API while it already
// lists; a Machine that is really gone is dropped by the GET below.
Effect.retry({
while: (error) => error._tag === "NotFound",
schedule: Schedule.spaced("1 second"),
times: 5,
}),
Effect.catchTag("NotFound", () => Effect.void),
);
}),
{ concurrency: 1 },
).pipe(Effect.timeout("30 seconds"));
const observed: FlyMachine[] = [];
for (const machine of ordered) {
if (!machine.id) continue;
if (machine.host_status === "unreachable") {
observed.push(machine);
continue;
}
const current = yield* leased.has(machine.id)
? getLeasedMachineById(appName, machine.id)
: getMachineById(appName, machine.id);
if (!current) {
yield* leases.forget(machine.id);
continue;
}
if (!sameOwnership(current, machine))
return yield* new ReplicaOwnershipChanged({ appName, machineId: machine.id });
yield* leases.checkTarget(machine.id);
observed.push(current);
}
yield* leases.check;
return observed;
});
export const retireMachines = (
appName: string,
snapshot: FlyMachine[],
replacementReady = false,
existingLeases?: MachineLeases,
) =>
usingMachineLeases(appName, existingLeases, (leases) =>
retireLeasedMachines(appName, snapshot, replacementReady, leases),
);
const retireLeasedMachines = Effect.fn(function* (
appName: string,
snapshot: FlyMachine[],
replacementReady: boolean,
leases: MachineLeases,
) {
const targets = yield* leaseSnapshot(appName, snapshot, leases);
const results = yield* Effect.forEach(
targets,
(machine) =>
Effect.gen(function* () {
yield* leases.check;
const result = yield* Effect.gen(function* () {
if (!machine.id) return;
if (replacementReady && machine.host_status === "unreachable") {
const metadata = machine.config?.metadata;
if (
!machine.config ||
machine.incomplete_config ||
(machine.config.mounts?.length ?? 0) > 0 ||
!metadata?.[alchemyMetadataKeys.instance] ||
!metadata[alchemyMetadataKeys.fqn]
) {
return yield* new ReplicaOwnershipChanged({ appName, machineId: machine.id });
}
yield* Effect.logWarning(
"Force-retiring owned stateless Fly Machine on unreachable host; work was not proven drained",
{ appName, machineId: machine.id },
);
const machineId = machine.id;
const heldNonce = yield* leases.nonceIfHeld(machineId);
yield* Effect.gen(function* () {
const remove = (lease_nonce?: string) =>
machines
.deleteMachine({
app_name: appName,
machine_id: machineId,
force: true,
lease_nonce,
})
.pipe(Retry.none, Effect.timeout("30 seconds"));
if (heldNonce !== undefined) yield* leases.remove(machineId, remove);
else yield* leases.guard(remove());
}).pipe(
Effect.catchTag("NotFound", () => Effect.void),
retirementStep(appName, machineId, "unreachable force-delete"),
);
yield* leases.forget(machine.id);
return yield* waitDestroyed(appName, machine.id);
}
yield* retireMachine(appName, machine.id, machine, leases);
}).pipe(Effect.result);
yield* leases.check;
return Result.isFailure(result)
? result.failure._tag === "Fly.ReplicaRetirementIncomplete"
? result.failure.residuals
: [{ machineId: machine.id ?? "", stage: result.failure._tag }]
: [];
}),
{ concurrency: 4 },
);
const residuals = results.flat();
if (residuals.length) return yield* new ReplicaRetirementIncomplete({ appName, residuals });
});
export const retireMachine = (
appName: string,
machineId: string,
snapshot?: FlyMachine,
existingLeases?: MachineLeases,
) =>
usingMachineLeases(appName, existingLeases, (leases) =>
retireLeasedMachine(appName, machineId, snapshot, leases).pipe(
Effect.tap(() => leases.forget(machineId)),
),
);
const retireLeasedMachine = Effect.fn(
function* (
appName: string,
machineId: string,
snapshot: FlyMachine | undefined,
leases: MachineLeases,
) {
yield* leases.acquire([machineId]);
const current = yield* getMachineById(appName, machineId);
if (current === undefined) return;
if (snapshot && !sameOwnership(current, snapshot)) {
return yield* new ReplicaOwnershipChanged({ appName, machineId });
}
const stop = yield* predecessorShutdown(current);
if (
current.cordoned === true &&
current.config?.metadata?.[alchemyMetadataKeys.phase] === "candidate"
)
return yield* deleteMachine(appName, machineId, leases);
if (current.state !== "stopped" && current.state !== "created") {
yield* leases
.mutate(
machineId,
(lease_nonce) =>
machines
.cordonMachine({ app_name: appName, machine_id: machineId, lease_nonce })
.pipe(Effect.timeout("30 seconds")),
{ idempotent: true },
)
.pipe(retirementStep(appName, machineId, "cordon"));
if (hasPublishedService(current.config?.services)) yield* Effect.sleep("10 seconds");
yield* leases
.mutate(
machineId,
(lease_nonce) =>
machines
.stopMachine({
app_name: appName,
machine_id: machineId,
lease_nonce,
signal: stop.signal,
timeout: stop.timeout,
})
.pipe(Effect.timeout(stop.timeoutMs + 15_000)),
{ timeoutMs: stop.timeoutMs + 120_000 },
)
.pipe(retirementStep(appName, machineId, "stop"));
yield* machines
.waitMachine({
app_name: appName,
machine_id: machineId,
state: "stopped",
instance_id: current.instance_id,
timeout: WAIT_TIMEOUT_SECONDS,
})
.pipe(
Retry.none,
Effect.retry({
times: 10,
schedule: Schedule.spaced(Math.max(500, stop.timeoutMs / 10)),
while: (error) =>
error._tag === "MachineWaitTimeout" || error._tag === "GatewayTimeout",
}),
Effect.timeout(stop.timeoutMs + 15_000),
retirementStep(appName, machineId, "wait-stopped"),
);
}
yield* leases
.remove(machineId, (lease_nonce) =>
machines
.deleteMachine({ app_name: appName, machine_id: machineId, force: false, lease_nonce })
.pipe(Effect.timeout("30 seconds")),
)
.pipe(
Effect.catchTag("NotFound", () => Effect.void),
retirementStep(appName, machineId, "delete"),
);
yield* waitDestroyed(appName, machineId).pipe(
retirementStep(appName, machineId, "verify-absent"),
);
},
Effect.catchTag("NotFound", () => Effect.void),
);
const mountedDisksOf = (machine: FlyMachine, volumesById: Map<string, FlyVolume>): MountedDisk[] =>
(machine.config?.mounts ?? []).flatMap((mount) => {
const volumeId = mount.volume;
const path = mount.path;
if (volumeId === undefined || path === undefined) return [];
const volume = volumesById.get(volumeId);
return [{ path, volumeId, sizeGb: volume?.size_gb ?? 0, name: volume?.name ?? "" }];
});
export const toReplica = (machine: FlyMachine, volumesById: Map<string, FlyVolume>): Replica => ({
machineId: machine.id ?? "",
name: machine.name ?? "",
region: machine.region ?? "",
state: machine.state ?? "",
instanceId: machine.instance_id,
privateIp: machine.private_ip,
imageRef: toImageRef(machine.image_ref),
guest: toGuestAttrs(machine.config?.guest),
mounts: mountedDisksOf(machine, volumesById),
});
export const toReplicaSet = (
replicas: Replica[],
appName: string,
baseName: string,
services?: FlyMachineService[],
): ReplicaSet => {
const primary = replicas[0];
return {
appName,
machineId: primary?.machineId ?? "",
machineIds: replicas.map((replica) => replica.machineId),
name: primary?.name ?? baseName,
baseName,
region: primary?.region ?? "",
regions: [
...new Set(replicas.map((replica) => replica.region).filter((region) => region.length > 0)),
],
state: primary?.state ?? "",
instanceId: primary?.instanceId,
privateIp: primary?.privateIp,
imageRef: primary?.imageRef,
guest: primary?.guest,
services,
count: replicas.length,
mounts: primary?.mounts ?? [],
replicas,
};
};
export const ownedReplicas = (
listed: FlyMachine[],
input: {
metadata: Record<string, string>;
resourceInstanceId: string;
fqn: string;
baseName?: string;
machineIds?: readonly string[];
},
) =>
listed.filter((machine) => {
const metadata = machine.config?.metadata ?? {};
if (!Object.entries(input.metadata).every(([key, value]) => metadata[key] === value))
return false;
if (metadata[alchemyMetadataKeys.instance] !== undefined) {
return (
metadata[alchemyMetadataKeys.instance] === input.resourceInstanceId &&
metadata[alchemyMetadataKeys.fqn] === input.fqn
);
}
return (
(machine.id !== undefined && input.machineIds?.includes(machine.id)) ||
(input.baseName !== undefined &&
(machine.name === input.baseName ||
machine.name === replicaMachineName(input.baseName, replicaIndexOf(machine), 2)))
);
});
export const listReplicas = Effect.fn(function* (input: {
appName: string;
id: string;
type: FlyAlchemyType;
}) {
const machines = yield* listMachinesByApp(input.appName);
return machines
.filter((machine) => isOwnedType(machine, input.type) && alchemyIdOf(machine) === input.id)
.sort((left, right) => replicaIndexOf(left) - replicaIndexOf(right));
});
export const listReplicaSets = Effect.fn(function* (type: FlyAlchemyType) {
const apps = yield* listOwnedApps();
const groups = yield* Effect.forEach(
apps,
(app) =>
listMachinesByApp(app.appName).pipe(
Effect.map((machines) => {
const owned = machines.filter((machine) => isOwnedType(machine, type));
const byId = new Map<string, FlyMachine[]>();
for (const machine of owned) {
const id = alchemyIdOf(machine);
if (id === undefined) continue;
const metadata = machine.config?.metadata;
const key = JSON.stringify([
metadata?.[alchemyMetadataKeys.stack],
metadata?.[alchemyMetadataKeys.stage],
metadata?.[alchemyMetadataKeys.fqn] ?? id,
metadata?.[alchemyMetadataKeys.instance],
metadata?.[alchemyMetadataKeys.generation],
]);
const group = byId.get(key) ?? [];
group.push(machine);
byId.set(key, group);
}
return [...byId.values()].map((group) => {
const sorted = [...group].sort(
(left, right) => replicaIndexOf(left) - replicaIndexOf(right),
);
const replicas = sorted.map((machine) => toReplica(machine, new Map()));
return toReplicaSet(
replicas,
app.appName,
replicas[0]?.name ?? "",
sorted[0]?.config?.services,
);
});
}),
),
{ concurrency: 8 },
);
return groups.flat();
});
const pickVolume = (
group: FlyVolume[],
used: Set<string>,
preferId: string | undefined,
): FlyVolume | undefined => {
if (preferId !== undefined && !used.has(preferId)) {
const preferred = group.find((volume) => volume.id === preferId);
if (preferred !== undefined) return preferred;
}
return group.find((volume) => volume.id !== undefined && !used.has(volume.id));
};
export interface ReconcileReplicasInput {
fqn: string;
resourceInstanceId: string;
policy: DeploymentPolicy;
checks?: Record<string, MachineCheck>;
id: string;
type: FlyAlchemyType;
appName: string;
baseName: string;
/**
* Regions to run in. `count` is the total replica count; replica `i`
* runs in `regions[i % regions.length]`.
*/
regions: readonly string[];
count: number;
disks: DiskSpec[];
skipLaunch?: boolean;
minSecretsVersion?: number;
outputMachineIds?: readonly string[];
preferVolumeIds?: ReadonlyArray<ReadonlyArray<string>>;
configDrifted: (
machine: FlyMachine,
desired: { mounts: FlyMachineMount[]; metadata: Record<string, string> },
) => boolean;
buildConfig: (replica: {
index: number;
mounts: FlyMachineMount[];
metadata: Record<string, string>;
}) => FlyMachineConfig;
}
export const reconcileReplicas = (input: ReconcileReplicasInput) =>
usingMachineLeases(input.appName, undefined, (leases) => reconcileLeasedReplicas(input, leases));
const reconcileLeasedReplicas = Effect.fn(function* (
input: ReconcileReplicasInput,
leases: MachineLeases,
) {
const ownership = yield* createMachineMetadata(input.id, input.type);
const alchemy = {
...ownership,
[alchemyMetadataKeys.instance]: input.resourceInstanceId,
[alchemyMetadataKeys.fqn]: input.fqn,
[alchemyMetadataKeys.baseName]: input.baseName,
};
const buildConfig: ReconcileReplicasInput["buildConfig"] = (replica) => ({
...input.buildConfig(replica),
checks:
input.checks === undefined
? undefined
: Object.fromEntries(
Object.entries(input.checks).map(([name, check]) => [name, toFlyServiceCheck(check)]),
),
stop_config:
input.policy.shutdown === undefined
? undefined
: { signal: input.policy.shutdown.signal, timeout: input.policy.shutdown.timeout },
});
const config = buildConfig({ index: 0, mounts: [], metadata: alchemy });
yield* validateDeployment(input.policy, config, input.disks.length > 0, input.skipLaunch);
if (input.policy.bluegreen)
return yield* reconcileBlueGreen({ ...input, buildConfig }, ownership, alchemy, leases);
const desiredNames = new Set(
Array.from({ length: input.count }, (_, index) =>
replicaMachineName(input.baseName, index, input.count),
),
);
const listed = yield* listMachinesByApp(input.appName);
const owned = yield* leaseSnapshot(
input.appName,
ownedReplicas(listed, { ...input, metadata: ownership, machineIds: input.outputMachineIds }),
leases,
);
const byIndex = new Map<number, FlyMachine>();
const preferIds = new Set((input.outputMachineIds ?? []).filter((id) => id.length > 0));
for (const machine of owned) {
const id = machine.id;
if (id !== undefined && preferIds.has(id)) {
byIndex.set(replicaIndexOf(machine), machine);
}
}
for (const machine of owned) {
const name = machine.name;
if (
name === undefined ||
(!desiredNames.has(name) &&
machine.config?.metadata?.[alchemyMetadataKeys.instance] !== input.resourceInstanceId)
)
continue;
const index = replicaIndexOf(machine);
if (!byIndex.has(index)) byIndex.set(index, machine);
}
const scaleDown = [...byIndex].filter(([index]) => index >= input.count);
yield* retireMachines(
input.appName,
scaleDown.map(([, machine]) => machine),
false,
leases,
);
for (const [index] of scaleDown) byIndex.delete(index);
// Volumes are regional: one group per disk and region, sized to the
// replicas that run there.
const replicasIn = (region: string) =>
Array.from({ length: input.count }, (_, index) => index).filter(
(index) => regionOfReplica(input.regions, index) === region,
).length;
const groups: Array<{
disk: DiskSpec;
name: string;
byRegion: Map<string, { volumes: FlyVolume[]; extras: FlyVolume[] }>;
}> = [];
for (const [diskIndex, disk] of input.disks.entries()) {
const name = yield* volumeGroupName(input.id, disk);
const preferIds = (input.preferVolumeIds ?? [])
.map((replica) => replica[diskIndex])
.filter((id): id is string => id !== undefined && id.length > 0);
const byRegion = new Map<string, { volumes: FlyVolume[]; extras: FlyVolume[] }>();
for (const region of input.regions) {
byRegion.set(
region,
yield* ensureVolumeGroup({
appName: input.appName,
name,
region,
count: replicasIn(region),
disk,
preferIds,
}),
);
}
groups.push({ disk, name, byRegion });
}
const usedVolumeIds = new Set<string>();
const live: FlyMachine[] = [];
for (let index = 0; index < input.count; index++) {
const name = replicaMachineName(input.baseName, index, input.count);
const region = regionOfReplica(input.regions, index);
const metadata = {
...alchemy,
[alchemyMetadataKeys.replica]: String(index),
...(input.minSecretsVersion === undefined
? {}
: { [alchemyMetadataKeys.secretsVersion]: String(input.minSecretsVersion) }),
};
const prefer = input.preferVolumeIds?.[index] ?? [];
const mounts: FlyMachineMount[] = [];
for (const [diskIndex, group] of groups.entries()) {
const volume = pickVolume(
group.byRegion.get(region)?.volumes ?? [],
usedVolumeIds,
prefer[diskIndex],
);
const volumeId = volume?.id;
if (volumeId === undefined) {
return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName });
}
usedVolumeIds.add(volumeId);
mounts.push({ volume: volumeId, path: group.disk.path });
}
const config = buildConfig({ index, mounts, metadata });
let current = byIndex.get(index);
// Fly cannot move a Machine between regions. Machine names are unique
// per App, so retire the old replica before creating it in its region.
if (current !== undefined && current.region !== region) {
yield* retireMachines(input.appName, [current], false, leases);
byIndex.delete(index);
current = undefined;
}
if (current === undefined) {
const created = yield* machines
.createMachine({
app_name: input.appName,
name,
region,
config,
skip_launch: input.skipLaunch === true ? true : undefined,
min_secrets_version: input.minSecretsVersion,
})
.pipe(Effect.catchTag("Conflict", () => Effect.undefined));
current =
created ??
(yield* listMachinesByApp(input.appName).pipe(
Effect.map((machines) =>
ownedReplicas(machines, {
...input,
metadata: ownership,
machineIds: input.outputMachineIds,
}).find((machine) => machine.name === name),
),
));
if (current === undefined || current.id === undefined) {
return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName });
}
const observed = (yield* leaseSnapshot(input.appName, [current], leases))[0];
if (!observed || input.configDrifted(observed, { mounts, metadata })) {
return yield* new ReplicaNotCreated({ resourceName: name, appName: input.appName });
}
current = observed;
} else if (
!sameChecks(current.config?.checks, config.checks) ||
!sameStopConfig(current.config?.stop_config, config.stop_config) ||
input.configDrifted(current, { mounts, metadata })
) {
const machineId = current.id!;
const currentVersion = current.instance_id;
const previous = current;
const expected = { ...previous, config };
yield* leases.mutate(machineId, (lease_nonce) =>
machines
.updateMachine({
app_name: input.appName,
machine_id: machineId,
current_version: currentVersion,
config,
skip_launch: input.skipLaunch === true ? true : undefined,
min_secrets_version: input.minSecretsVersion,
lease_nonce,
})
.pipe(Effect.timeout("30 seconds")),
);
// Rolling opt-out removes generation metadata; observe that transition before startup.
current = yield* leases.guard(
Effect.gen(function* () {
const observed = yield* machines
.getMachine({ app_name: input.appName, machine_id: machineId })
.pipe(Retry.none);
if (!sameOwnership(observed, expected) && !sameOwnership(observed, previous))
return yield* new ReplicaOwnershipChanged({ appName: input.appName, machineId });
if (
!sameOwnership(observed, expected) ||
input.configDrifted(observed, { mounts, metadata }) ||
!sameChecks(observed.config?.checks, config.checks) ||
!sameStopConfig(observed.config?.stop_config, config.stop_config)
)
return yield* new ReplicaNotCreated({ appName: input.appName, resourceName: name });
return observed;
}).pipe(
Effect.retry({
times: 8,
schedule: waitBackoff,
while: (error) =>
error._tag === "Fly.ReplicaNotCreated" ||
TRANSIENT_GET_TAGS.some((tag) => tag === error._tag),
}),
Effect.timeout("60 seconds"),
),
);
}
current = yield* ensureStarted(
input.appName,
current,
input.skipLaunch === true,
input.policy.healthTimeoutMs,
config,
leases,
);
if (current.cordoned === true && !input.skipLaunch) {
yield* setRouting(input.appName, current.id!, false, leases);
if (hasPublishedService(config.services)) yield* Effect.sleep("10 seconds");
current = yield* waitHealthy(input.appName, current, input.policy.healthTimeoutMs, config);
}
live.push(current);
}
const liveIds = new Set(live.map((machine) => machine.id));
yield* retireMachines(
input.appName,
owned.filter((machine) => !liveIds.has(machine.id)),
false,
leases,
);
for (const group of groups) {
for (const extra of [...group.byRegion.values()].flatMap((placed) => placed.extras)) {
const volumeId = extra.id;
if (volumeId === undefined || usedVolumeIds.has(volumeId)) continue;
yield* deleteVolume(input.appName, volumeId);
}
}
const volumesById = new Map<string, FlyVolume>();
for (const group of groups) {
for (const volume of [...group.byRegion.values()].flatMap((placed) => placed.volumes)) {
if (volume.id !== undefined) volumesById.set(volume.id, volume);
}
}
const fresh = yield* Effect.forEach(
live,
(machine) =>
machine.id === undefined
? Effect.succeed(machine)
: getMachineById(input.appName, machine.id).pipe(Effect.map((next) => next ?? machine)),
{ concurrency: 4 },
);
const replicas = fresh.map((machine) => toReplica(machine, volumesById));
return toReplicaSet(replicas, input.appName, input.baseName, fresh[0]?.config?.services);
});
export const deleteReplicaSet = Effect.fn(function* (input: {
appName: string;
id: string;
type: FlyAlchemyType;
fqn: string;
resourceInstanceId: string;
force?: boolean;
machineIds: readonly string[];
volumeIds: readonly string[];
}) {
return yield* usingMachineLeases(input.appName, undefined, (leases) =>
Effect.gen(function* () {
const owned = yield* leaseSnapshot(
input.appName,
ownedReplicas(yield* listMachinesByApp(input.appName), {
...input,
metadata: yield* createMachineMetadata(input.id, input.type),
}),
leases,
);
if (input.force) {
yield* Effect.forEach(
owned,
(machine) =>
Effect.gen(function* () {
if (!machine.id) return;
const current = yield* getMachineById(input.appName, machine.id);
if (!current) return;
if (!sameOwnership(current, machine))
return yield* new ReplicaOwnershipChanged({
appName: input.appName,
machineId: machine.id,
});
yield* deleteMachine(input.appName, machine.id, leases);
}),
{ concurrency: 4 },
);
} else yield* retireMachines(input.appName, owned, false, leases);
yield* Effect.forEach(
[...new Set(input.volumeIds)].filter((id) => id.length > 0),
(volumeId) => deleteVolume(input.appName, volumeId),
{ concurrency: 4 },
);
}),
);
});
export const volumeIdsOf = (set: {
mounts?: readonly MountedDisk[];
replicas?: readonly Replica[];
}): string[] => {
const ids = new Set<string>();
for (const mount of set.mounts ?? []) ids.add(mount.volumeId);
for (const replica of set.replicas ?? []) {
for (const mount of replica.mounts) ids.add(mount.volumeId);
}
return [...ids];
};
export const groupGenerations = (machines: FlyMachine[]) => {
const groups = new Map<string | undefined, FlyMachine[]>();
for (const machine of machines) {
const generation = machine.config?.metadata?.[alchemyMetadataKeys.generation];
const group = groups.get(generation) ?? [];
group.push(machine);
groups.set(generation, group);
}
return groups;
};
export const observeReplicaSet = Effect.fn(function* (input: {
appName?: string;
fqn: string;
resourceInstanceId: string;
id: string;
type: FlyAlchemyType;
machineIds?: readonly string[];
baseName?: string;
}) {
if (input.appName === undefined) return undefined;
const owned = ownedReplicas(yield* listMachinesByApp(input.appName), {
...input,
metadata: yield* createMachineMetadata(input.id, input.type),
});
const groups = groupGenerations(owned);
const committed = [...groups.entries()]
.filter(([generation, group]) => {
if (generation === undefined) return false;
const count = Number(group[0]?.config?.metadata?.[alchemyMetadataKeys.count]);
if (group.some((machine) => classifyDeploymentState(machine).protocol === "invalid"))
return false;
const protocol2 = group.some(
(machine) => machine.config?.metadata?.[alchemyMetadataKeys.protocol] === "2",
);
if (protocol2 && !validProtocol2Generation(group)) return false;
return (
Number.isSafeInteger(count) &&
count > 0 &&
group.length === count &&
new Set(group.map(replicaIndexOf)).size === count &&
group.every((machine) => {
const metadata = machine.config?.metadata;
const index = replicaIndexOf(machine);
return (
index >= 0 &&
index < count &&
Number(metadata?.[alchemyMetadataKeys.count]) === count &&
metadata?.[alchemyMetadataKeys.phase] === "active" &&
machine.cordoned === false &&
(metadata[alchemyMetadataKeys.protocol] === undefined ||
((metadata[alchemyMetadataKeys.protocol] === "1" ||
metadata[alchemyMetadataKeys.protocol] === "2") &&
metadata[alchemyMetadataKeys.restored] === "true" &&
(metadata[alchemyMetadataKeys.role] === "idle" ||
(machine.instance_id !== undefined &&
metadata[alchemyMetadataKeys.checkedInstance] === machine.instance_id))))
);
})
);
})
.sort(
([, a], [, b]) =>
Number(b[0]?.config?.metadata?.[alchemyMetadataKeys.sequence] ?? 0) -
Number(a[0]?.config?.metadata?.[alchemyMetadataKeys.sequence] ?? 0),
);
const legacy = (groups.get(undefined) ?? []).filter((machine) => {
const { protocol } = classifyDeploymentState(machine);
return protocol === "legacy" || protocol === "1";
});
const listed = (committed[0]?.[1] ?? legacy).sort(
(a, b) => replicaIndexOf(a) - replicaIndexOf(b),
);
const rolloutPending = owned.some((machine) => !listed.includes(machine));
if (listed.length > 0) {
const volumesById = new Map<string, FlyVolume>();
for (const machine of listed) {
for (const mount of machine.config?.mounts ?? []) {
const volumeId = mount.volume;
if (volumeId === undefined || volumesById.has(volumeId)) continue;
const volume = yield* getVolumeById(input.appName, volumeId);
if (volume !== undefined) volumesById.set(volumeId, volume);
}
}
const replicas = listed.map((machine) => toReplica(machine, volumesById));
return {
...toReplicaSet(
replicas,
input.appName,
listed[0]?.config?.metadata?.[alchemyMetadataKeys.baseName] ??
input.baseName ??
replicas[0]?.name ??
"",
listed[0]?.config?.services,
),
rolloutPending,
};
}
if (owned.length)
return {
...toReplicaSet([], input.appName, input.baseName ?? ""),
region: owned[0]?.region ?? "",
regions: [
...new Set(
owned.map((machine) => machine.region ?? "").filter((region) => region.length > 0),
),
],
rolloutPending: true,
};
return undefined;
});