import * as machines from "@distilled.cloud/fly-io/machines";
import type { Machine } from "@distilled.cloud/fly-io/machines";
import * as Retry from "@distilled.cloud/fly-io/Retry";
import { deepEqual } from "alchemy/Diff";
import { sha256Object } from "alchemy/Util/sha256";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import {
applyImageSet,
encodeImageSet,
pinsFromConfig,
sameImageSet,
validObservedImageSet,
} from "./DeploymentImages.ts";
import { classifyDeploymentState, validProtocol2Generation } from "./DeploymentState.ts";
import { usingMachineLeases, type MachineLeases } from "./leases.ts";
import { canonicalContainers } from "./MachineContainers.ts";
import { alchemyMetadataKeys as keys } from "./Metadata.ts";
import { regionOfReplica } from "./Region.ts";
import {
autostopMode,
checksPassing,
deleteMachine,
ensureStarted,
getMachineById,
groupGenerations,
hasPublishedService,
listMachinesByApp,
leaseSnapshot,
ownedReplicas,
ReplicaNotCreated,
replicaIndexOf,
retireMachines,
sameChecks,
sameServices,
sameStopConfig,
toReplica,
toReplicaSet,
waitHealthy,
type ReconcileReplicasInput,
} from "./replicas.ts";
const metadataOf = (machine: Machine) =>
Object.fromEntries(
Object.entries(machine.config?.metadata ?? {}).flatMap(([key, value]) =>
value === undefined ? [] : [[key, value]],
),
);
const setMetadata = Effect.fn(function* (
appName: string,
machine: Machine,
metadata: Record<string, string>,
leases: MachineLeases,
) {
yield* leases.checkTarget(machine.id!);
// Metadata is not nonce-fenced by Fly; it is recovery evidence, not a lock.
yield* leases.mutate(
machine.id!,
() =>
machines
.patchMachineMetadata({ app_name: appName, machine_id: machine.id!, metadata })
.pipe(Effect.timeout("30 seconds")),
{ idempotent: true },
);
yield* Effect.sync(() => {
machine.config = { ...machine.config, metadata };
});
});
const phase = Effect.fn(function* (
appName: string,
machine: Machine,
value: string,
leases: MachineLeases,
) {
if (machine.config?.metadata?.[keys.phase] !== value) {
yield* setMetadata(appName, machine, { ...metadataOf(machine), [keys.phase]: value }, leases);
}
});
export const setRouting = (
appName: string,
machineId: string,
cordoned: boolean,
existingLeases?: MachineLeases,
) =>
usingMachineLeases(appName, existingLeases, (leases) =>
Effect.gen(function* () {
yield* leases.acquire([machineId]);
yield* leases.mutate(
machineId,
(lease_nonce) =>
Effect.gen(function* () {
const request = { app_name: appName, machine_id: machineId, lease_nonce };
if (cordoned) yield* machines.cordonMachine(request);
else yield* machines.uncordonMachine(request);
}).pipe(Effect.timeout("30 seconds")),
{ idempotent: true },
);
yield* getMachineById(appName, machineId).pipe(
Effect.filterOrFail(
(machine) => machine !== undefined && machine.cordoned === cordoned,
() => new ReplicaNotCreated({ appName, resourceName: machineId }),
),
Effect.retry({
times: 8,
schedule: Schedule.spaced("500 millis"),
while: (error) => error._tag === "Fly.ReplicaNotCreated",
}),
);
}),
);
const pinnedImage = (machine: Machine) => {
const image = machine.image_ref;
if (!image?.digest || !image.repository) return undefined;
return `${image.registry ? `${image.registry}/` : ""}${image.repository}@${image.digest}`;
};
export class DeploymentRecoveryAmbiguous extends Data.TaggedError(
"Fly.DeploymentRecoveryAmbiguous",
)<{ appName: string; message: string }> {}
export const readinessRoles = (
config: machines.FlyMachineConfig,
count: number,
predecessors: Machine[],
): Array<"run" | "idle"> => {
const services = config.services ?? [];
if (!services.length || services.some((service) => autostopMode(service.autostop) === "off")) {
return Array.from({ length: count }, () => "run");
}
const roles: Array<"run" | "idle"> = Array.from({ length: count }, (_, index) =>
predecessors.some((machine) => replicaIndexOf(machine) === index && machine.state === "started")
? "run"
: "idle",
);
const floor = Math.min(
count,
Math.max(1, ...services.map((service) => service.min_machines_running ?? 0)),
);
for (let index = 0; roles.filter((role) => role === "run").length < floor; index++)
roles[index] = "run";
return roles;
};
const idle = (machine: Machine) =>
machine.state === "stopped" || machine.state === "suspended" || machine.state === "created";
/** Cloud metadata records preparation and promotion; replica zero commits a complete set. */
export const reconcileBlueGreen = Effect.fn(function* (
input: ReconcileReplicasInput,
ownership: Record<string, string>,
metadata: Record<string, string>,
leases: MachineLeases,
) {
const config = input.buildConfig({ index: 0, mounts: [], metadata });
const containerPins = config.containers === undefined ? undefined : pinsFromConfig(config);
const containerMode = config.containers !== undefined;
const ambiguous = (message: string) =>
new DeploymentRecoveryAmbiguous({ appName: input.appName, message });
if (containerMode && containerPins === undefined)
return yield* ambiguous("Blue/green requires a complete immutable container image set.");
const workload = yield* sha256Object({
config: containerMode
? { ...config, containers: canonicalContainers(config.containers) }
: config,
count: input.count,
minSecretsVersion: input.minSecretsVersion,
// A single region hashes as before, so existing generations stay current.
region: input.regions.length === 1 ? input.regions[0] : [...input.regions],
});
const observe = listMachinesByApp(input.appName).pipe(
Effect.flatMap((listed) => {
const owned = ownedReplicas(listed, {
...input,
metadata: ownership,
machineIds: input.outputMachineIds,
});
const changed = listed.find(
(machine) =>
machine.id && input.outputMachineIds?.includes(machine.id) && !owned.includes(machine),
);
return changed
? Effect.fail(
ambiguous(`Cached Machine ${changed.id} no longer has the expected ownership.`),
)
: Effect.succeed(owned);
}),
);
const snapshot = yield* observe;
const owned = yield* leaseSnapshot(input.appName, snapshot, leases);
// A vanished snapshot member can have a successor outside this inventory.
if (snapshot.some((machine) => !owned.some((current) => current.id === machine.id))) {
return yield* ambiguous(
"Owned Machine membership changed while acquiring leases; retry with a fresh deployment snapshot.",
);
}
for (const machine of owned) {
const state = classifyDeploymentState(machine);
if (state.protocol === "invalid")
return yield* ambiguous(`Machine ${machine.id}: ${state.reason}`);
}
const idleAllowed =
(config.services?.length ?? 0) > 0 &&
config.services!.every((service) => autostopMode(service.autostop) !== "off");
const preparationServices = config.services?.map((service) => ({ ...service, autostop: "off" }));
const mismatchOf = (machine: Machine): string | undefined => {
const observed = metadataOf(machine);
if (containerMode) {
if (
observed[keys.protocol] !== "2" ||
!validObservedImageSet(machine) ||
!sameImageSet(pinsFromConfig(machine.config), containerPins)
)
return "container image set";
} else {
const pinned = observed[keys.image];
if (pinned === undefined && observed[keys.phase] !== "candidate") return "metadata.image";
if (!pinnedImage(machine) || (pinned !== undefined && pinned !== pinnedImage(machine)))
return "image_ref";
}
const secretsVersion = Number(observed[keys.secretsVersion] ?? -1);
if (
input.minSecretsVersion !== undefined &&
(!Number.isSafeInteger(secretsVersion) || secretsVersion < input.minSecretsVersion)
)
return "metadata.secretsVersion";
const temporary =
observed[keys.role] === "run" &&
observed[keys.restored] === "false" &&
sameServices(machine.config?.services, preparationServices);
if (observed[keys.workload] !== workload) return "metadata.workload";
if (
input.configDrifted(
{
...machine,
config: {
...machine.config,
image: config.image,
services: temporary ? config.services : machine.config?.services,
},
},
{
mounts: [],
metadata: {
...Object.fromEntries(
Object.entries(observed).filter(([key]) => key.startsWith("alchemy.")),
),
...metadata,
},
},
)
)
return "config";
if (!sameChecks(machine.config?.checks, config.checks)) return "config.checks";
if (!sameStopConfig(machine.config?.stop_config, config.stop_config))
return "config.stop_config";
return undefined;
};
const matches = (machine: Machine) => mismatchOf(machine) === undefined;
const groups = groupGenerations(owned);
const sequences = new Set<string>();
for (const [generation, group] of groups) {
if (generation === undefined) continue;
const sequence = group[0]?.config?.metadata?.[keys.sequence];
if (
!sequence ||
!Number.isSafeInteger(Number(sequence)) ||
Number(sequence) < 1 ||
sequences.has(sequence) ||
group.some((machine) => machine.config?.metadata?.[keys.sequence] !== sequence) ||
(group.some((machine) => machine.config?.metadata?.[keys.protocol] === "2") &&
!validProtocol2Generation(group))
) {
return yield* ambiguous(
"Owned generations have ambiguous sequence/lineage metadata; preserving all capacity.",
);
}
sequences.add(sequence);
}
const reusable = [...groups.entries()]
.filter(([generation, group]) => generation !== undefined && group.every(matches))
.sort(
([, a], [, b]) =>
Number(b[0]?.config?.metadata?.[keys.sequence] ?? 0) -
Number(a[0]?.config?.metadata?.[keys.sequence] ?? 0),
)[0];
const generation =
reusable?.[0] ??
(yield* sha256Object({
workload,
predecessors: owned
.map((machine) => machine.id)
.sort((a, b) => ((a ?? "") < (b ?? "") ? -1 : (a ?? "") > (b ?? "") ? 1 : 0)),
})).slice(0, 20);
const desired = reusable?.[1] ?? [];
if (
new Set(desired.map(replicaIndexOf)).size !== desired.length ||
desired.some((machine) => replicaIndexOf(machine) >= input.count)
) {
return yield* ambiguous(
"The recovered generation has duplicate or out-of-range replica indices.",
);
}
const sequence =
desired[0]?.config?.metadata?.[keys.sequence] ??
String(
1 +
Math.max(
0,
...owned.map((machine) => Number(machine.config?.metadata?.[keys.sequence] ?? 0)),
),
);
const predecessors = owned.filter(
(machine) => machine.config?.metadata?.[keys.generation] !== generation,
);
const recordedRoles = desired[0]?.config?.metadata?.[keys.roles];
const roles = recordedRoles?.split(",") ?? readinessRoles(config, input.count, predecessors);
if (
roles.length !== input.count ||
roles.some((role) => role !== "run" && role !== "idle") ||
!roles.includes("run") ||
desired.some(
(machine) =>
machine.config?.metadata?.[keys.roles] !== recordedRoles ||
((machine.config?.metadata?.[keys.protocol] === "1" ||
machine.config?.metadata?.[keys.protocol] === "2") &&
machine.config.metadata[keys.role] !== roles[replicaIndexOf(machine)]),
)
) {
return yield* ambiguous("The recovered generation has inconsistent readiness roles.");
}
const complete =
desired.length === input.count &&
desired.every(
(machine) =>
machine.config?.metadata?.[keys.phase] === "active" &&
machine.config.metadata[keys.restored] === "true" &&
(roles[replicaIndexOf(machine)] === "idle" ||
(machine.instance_id !== undefined &&
machine.config.metadata[keys.checkedInstance] === machine.instance_id)) &&
machine.cordoned === false,
);
if (
complete &&
!predecessors.length &&
desired.every((machine) => machine.state === "started" || (idleAllowed && idle(machine)))
) {
return toReplicaSet(
[...desired]
.sort((a, b) => replicaIndexOf(a) - replicaIndexOf(b))
.map((machine) => toReplica(machine, new Map())),
input.appName,
input.baseName,
config.services,
);
}
const candidates: Machine[] = [];
const snapshotIds = new Set(owned.map((machine) => machine.id));
let image = desired[0] ? pinnedImage(desired[0]) : undefined;
const order = Array.from({ length: input.count }, (_, index) => index).sort(
(a, b) => Number(roles[b] === "run") - Number(roles[a] === "run") || a - b,
);
// Creation-time predecessor IDs are diagnostic lineage, not deletion authority.
const predecessorIds = predecessors
.flatMap((machine) => (machine.id ? [machine.id] : []))
.sort()
.join(",");
yield* Effect.gen(function* () {
for (const index of order) {
const suffix = `-${generation}-${index}`;
const name = `${input.baseName.slice(0, 30 - suffix.length).replace(/-+$/g, "")}${suffix}`;
let current = desired.find((machine) => replicaIndexOf(machine) === index);
if (current === undefined) {
const run = roles[index] === "run";
const candidateConfig = input.buildConfig({
index,
mounts: [],
metadata: {
...metadata,
[keys.replica]: String(index),
[keys.generation]: generation,
[keys.workload]: workload,
[keys.phase]: "candidate",
[keys.sequence]: sequence,
[keys.count]: String(input.count),
[keys.protocol]: containerMode ? "2" : "1",
[keys.roles]: roles.join(","),
[keys.role]: roles[index]!,
[keys.predecessors]: predecessorIds,
[keys.restored]: String(!run || sameServices(config.services, preparationServices)),
...(input.minSecretsVersion === undefined
? {}
: { [keys.secretsVersion]: String(input.minSecretsVersion) }),
...(containerPins
? { [keys.containerImageSet]: encodeImageSet(containerPins)! }
: image
? { [keys.image]: image }
: {}),
},
});
const pinnedConfig = containerPins
? applyImageSet(candidateConfig, containerPins)
: candidateConfig;
if (pinnedConfig === undefined) return yield* ambiguous("Incomplete candidate image set.");
const readback = observe.pipe(
Effect.map((listed) =>
listed.find(
(machine) =>
machine.name === name &&
machine.config?.metadata?.[keys.generation] === generation &&
replicaIndexOf(machine) === index &&
matches(machine),
),
),
Effect.filterOrFail(
(machine) => machine !== undefined,
() => new ReplicaNotCreated({ appName: input.appName, resourceName: name }),
),
Effect.retry({
times: 8,
schedule: Schedule.spaced("500 millis"),
while: (error) => error._tag === "Fly.ReplicaNotCreated",
}),
);
current = yield* machines
.createMachine({
app_name: input.appName,
name,
region: regionOfReplica(input.regions, index),
config: {
...pinnedConfig,
image: containerPins ? undefined : (image ?? config.image),
services: run ? preparationServices : config.services,
},
skip_launch: !run,
skip_service_registration: true,
min_secrets_version: input.minSecretsVersion,
})
.pipe(
Retry.none,
Effect.timeout("30 seconds"),
Effect.catchTag(["Conflict", "GatewayTimeout", "TimeoutError"], () => readback),
Effect.catchReason("HttpClientError", "TransportError", () => readback),
);
}
if (!current?.id)
return yield* new ReplicaNotCreated({ appName: input.appName, resourceName: name });
const leased = (yield* leaseSnapshot(input.appName, [current], leases))[0];
if (!leased?.id)
return yield* ambiguous(
`Candidate ${current.id} disappeared before its lease was acquired.`,
);
const mismatch = mismatchOf(leased);
if (mismatch !== undefined)
return yield* ambiguous(
`Candidate ${current.id} changed before its lease was acquired. Mismatch: ${mismatch}.`,
);
current = leased;
snapshotIds.add(current.id);
if (containerPins) {
if (
!validObservedImageSet(current) ||
!sameImageSet(pinsFromConfig(current.config), containerPins)
)
return yield* ambiguous(`Container image pin mismatch on ${current.id}.`);
} else {
const resolvedImage = pinnedImage(current);
if (!resolvedImage || (image !== undefined && image !== resolvedImage))
return yield* ambiguous(`Image pin mismatch on ${current.id}.`);
image = resolvedImage;
if (current.config?.metadata?.[keys.image] !== image)
yield* setMetadata(
input.appName,
current,
{ ...metadataOf(current), [keys.image]: image },
leases,
);
}
const checked =
current.config?.metadata?.[keys.checkedInstance] === current.instance_id &&
current.instance_id !== undefined;
if (roles[index] === "run" && !(idleAllowed && checked && idle(current))) {
current = yield* ensureStarted(
input.appName,
current,
false,
input.policy.healthTimeoutMs,
current.config,
leases,
);
} else if (!idle(current)) {
current = yield* waitHealthy(
input.appName,
current,
input.policy.healthTimeoutMs,
current.config,
);
}
candidates[index] = current;
}
}).pipe(
Effect.catch((error) =>
Effect.gen(function* () {
const found = (yield* observe).filter(
(machine) =>
snapshotIds.has(machine.id) &&
machine.config?.metadata?.[keys.generation] === generation,
);
// A possibly promoted candidate is capacity, not rollback garbage.
if (
found.every(
(machine) =>
(!containerMode || matches(machine)) &&
machine.config?.metadata?.[keys.phase] === "candidate" &&
machine.cordoned === true,
)
) {
yield* Effect.forEach(
found,
(machine) =>
deleteMachine(input.appName, machine.id!, leases).pipe(
Effect.catch((cleanup) =>
Effect.logWarning("Fly candidate cleanup remains pending", {
appName: input.appName,
machineId: machine.id,
error: cleanup._tag,
}),
),
),
{ concurrency: 4 },
);
}
return yield* error;
}),
),
);
const needsPromotion = candidates.some(
(machine) => machine.cordoned !== false || machine.config?.metadata?.[keys.phase] !== "active",
);
if (needsPromotion) {
for (const machine of candidates) yield* phase(input.appName, machine, "promoting", leases);
// Promotion metadata resets reports; validate the whole set before routing any member.
for (const [index, machine] of candidates.entries()) {
const current = yield* getMachineById(input.appName, machine.id!);
if (!current || !matches(current))
return yield* ambiguous(`Candidate ${machine.id} changed before promotion.`);
if (current.state === "started") {
candidates[index] = yield* waitHealthy(
input.appName,
current,
input.policy.healthTimeoutMs,
current.config,
);
} else if (
!idleAllowed ||
!idle(current) ||
(roles[index] === "run" &&
(current.instance_id === undefined ||
current.config?.metadata?.[keys.checkedInstance] !== current.instance_id))
) {
return yield* ambiguous(`Candidate ${machine.id} has no readiness proof before promotion.`);
} else candidates[index] = current;
}
for (const machine of candidates)
if (machine.cordoned !== false) yield* setRouting(input.appName, machine.id!, false, leases);
if (hasPublishedService(config.services)) yield* Effect.sleep("10 seconds");
}
for (const [index, machine] of candidates.entries()) {
let current = machine;
if (current.config?.metadata?.[keys.restored] === "false") {
// Restoring services creates a new instance; its predecessor's checks cannot prove readiness.
const machineId = current.id!;
const currentVersion = current.instance_id;
const restoreConfig = input.buildConfig({
index,
mounts: [],
metadata: { ...metadataOf(current), [keys.restored]: "true" },
});
const restored = containerPins
? applyImageSet(restoreConfig, containerPins)
: { ...restoreConfig, image };
if (restored === undefined)
return yield* ambiguous("Incomplete image set during service restoration.");
current = yield* leases.mutate(machineId, (lease_nonce) =>
machines
.updateMachine({
app_name: input.appName,
machine_id: machineId,
lease_nonce,
current_version: currentVersion,
config: restored,
min_secrets_version: input.minSecretsVersion,
})
.pipe(Effect.timeout("30 seconds")),
);
current = yield* ensureStarted(
input.appName,
current,
false,
input.policy.healthTimeoutMs,
config,
leases,
);
} else {
current = (yield* getMachineById(input.appName, current.id!)) ?? current;
if (current.instance_id !== machine.instance_id) {
current = yield* ensureStarted(
input.appName,
current,
false,
input.policy.healthTimeoutMs,
config,
leases,
);
}
if (
current.state !== "started" &&
roles[index] === "run" &&
current.config?.metadata?.[keys.checkedInstance] !== current.instance_id
) {
current = yield* ensureStarted(
input.appName,
current,
false,
input.policy.healthTimeoutMs,
config,
leases,
);
}
}
candidates[index] = current;
}
// Reset reports while still pending; an interrupted validation is not a commit.
for (const machine of candidates) yield* phase(input.appName, machine, "validating", leases);
for (const [index, machine] of candidates.entries()) {
const current = yield* getMachineById(input.appName, machine.id!);
if (!current || !matches(current) || current.config?.metadata?.[keys.restored] !== "true")
return yield* ambiguous(`Candidate ${machine.id} changed before commitment.`);
if (current.state === "started")
candidates[index] = yield* waitHealthy(
input.appName,
current,
input.policy.healthTimeoutMs,
config,
);
else if (
!idleAllowed ||
!idle(current) ||
(roles[index] === "run" &&
(current.instance_id === undefined ||
(current.config?.metadata?.[keys.checkedInstance] !== current.instance_id &&
!(machine.instance_id === current.instance_id && checksPassing(machine, config)))))
) {
return yield* ambiguous(
`Candidate ${machine.id} has no readiness proof for its current instance.`,
);
} else candidates[index] = current;
}
// All final checks precede the commit stamps, with replica zero last.
for (const machine of [...candidates].reverse()) {
const committed = {
...metadataOf(machine),
[keys.phase]: "active",
...(machine.state === "started" || roles[replicaIndexOf(machine)] === "run"
? { [keys.checkedInstance]: machine.instance_id! }
: {}),
};
if (!deepEqual(metadataOf(machine), committed))
yield* setMetadata(input.appName, machine, committed, leases);
}
// Legacy ownership is required; predecessor phase stamps are only advisory.
for (const machine of predecessors) {
if (machine.config?.metadata?.[keys.protocol] === "2") {
const observed = yield* getMachineById(input.appName, machine.id!);
if (
!observed ||
observed.config?.metadata?.[keys.protocol] !== "2" ||
!validObservedImageSet(observed) ||
!sameImageSet(pinsFromConfig(observed.config), pinsFromConfig(machine.config))
)
return yield* ambiguous(
`Predecessor ${machine.id} image identity changed before retirement.`,
);
}
if (machine.config?.metadata?.[keys.instance] === undefined)
yield* setMetadata(input.appName, machine, { ...metadataOf(machine), ...metadata }, leases);
if (machine.host_status !== "unreachable")
yield* phase(input.appName, machine, "retiring", leases).pipe(
Effect.catch((error) =>
Effect.logWarning("Fly predecessor phase metadata was not updated", {
appName: input.appName,
machineId: machine.id,
error: error._tag,
}),
),
);
}
yield* retireMachines(input.appName, predecessors, true, leases);
return toReplicaSet(
candidates.map((machine) => toReplica(machine, new Map())),
input.appName,
input.baseName,
config.services,
);
});