import {
BadGateway,
GatewayTimeout,
InternalServerError,
ServiceUnavailable,
TooManyRequests,
} from "@distilled.cloud/fly-io/Errors";
import * as machines from "@distilled.cloud/fly-io/machines";
import * as Retry from "@distilled.cloud/fly-io/Retry";
import * as Clock from "effect/Clock";
import * as Data from "effect/Data";
import * as Deferred from "effect/Deferred";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as FetchHttpClient from "effect/http/FetchHttpClient";
import * as HttpClient from "effect/http/HttpClient";
import * as HttpClientRequest from "effect/http/HttpClientRequest";
import * as Option from "effect/Option";
import * as Schedule from "effect/Schedule";
const TTL_SECONDS = 120;
const EXPIRY_MARGIN_MS = 15_000;
const retrySchedule = <E>(): Schedule.Schedule<Duration.Duration, E> =>
Schedule.exponential("500 millis").pipe(
Schedule.modifyDelay<Duration.Duration, E>(({ input, duration }) =>
Effect.succeed(
(TooManyRequests.is(input) ||
InternalServerError.is(input) ||
BadGateway.is(input) ||
ServiceUnavailable.is(input) ||
GatewayTimeout.is(input)) &&
input.retryAfter
? Math.min(Duration.toMillis(input.retryAfter), 60_000)
: Math.min(Duration.toMillis(duration), 5_000),
),
),
Schedule.addDelay<Duration.Duration, E>(() =>
Effect.sync(() => Duration.millis(Math.random() * 50)),
),
);
export class MachineLeaseBusy extends Data.TaggedError("Fly.MachineLeaseBusy")<{
appName: string;
machineId: string;
}> {}
export class MachineLeaseLost extends Data.TaggedError("Fly.MachineLeaseLost")<{
appName: string;
machineId: string;
reason: string;
}> {
get message() {
return `Machine lease authority lost for ${this.appName}/${this.machineId}: ${this.reason}`;
}
}
export class MachineMutationUncertain extends Data.TaggedError("Fly.MachineMutationUncertain")<{
appName: string;
machineId: string;
}> {}
interface HeldLease {
nonce: string;
deadline: number;
renewAt: number;
}
export const makeMachineLeases = Effect.fn(function* (appName: string) {
const scope = yield* Effect.scope;
const held = new Map<string, HeldLease>();
const deleting = new Set<string>();
const renewals = new Set<Fiber.Fiber<void, never>>();
const uncertain = new Set<string>();
const lost = yield* Deferred.make<never, MachineLeaseLost>();
let failure: MachineLeaseLost | undefined;
const lose = (machineId: string, reason: string) =>
Effect.gen(function* () {
failure ??= new MachineLeaseLost({ appName, machineId, reason });
yield* Deferred.fail(lost, failure);
return yield* failure;
});
const check = Effect.gen(function* () {
if (failure) return yield* failure;
const now = yield* Clock.currentTimeMillis;
for (const [machineId, lease] of held) {
if (now >= lease.deadline) return yield* lose(machineId, "lease authority expired");
}
});
const guard = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
check.pipe(Effect.andThen(Effect.raceFirst(effect, Deferred.await(lost))));
const record = (machineId: string, response: machines.MachineLease, requestedAt: number) =>
Effect.gen(function* () {
const lease = response.data;
const now = yield* Clock.currentTimeMillis;
const deadline =
Math.min((lease?.expires_at ?? 0) * 1000, requestedAt + TTL_SECONDS * 1000) -
EXPIRY_MARGIN_MS;
const captured = lease?.nonce
? { nonce: lease.nonce, deadline, renewAt: requestedAt + 25_000 }
: undefined;
if (captured) held.set(machineId, captured);
if (!captured || !Number.isFinite(deadline) || deadline <= now) {
return yield* lose(machineId, "invalid or expired lease response");
}
return captured;
});
const acquire = (machineIds: readonly string[]) =>
Effect.gen(function* () {
for (const machineId of [...new Set(machineIds)].sort()) {
yield* check;
if (held.has(machineId)) continue;
// Save a returned nonce atomically; uncertain interrupted requests expire at Fly.
yield* Effect.uninterruptibleMask((restore) =>
Effect.gen(function* () {
const requestedAt = yield* Clock.currentTimeMillis;
const response = yield* restore(
machines
.createMachineLease({
app_name: appName,
machine_id: machineId,
ttl: TTL_SECONDS,
description: "Alchemy Machine lifecycle",
})
.pipe(
Retry.none,
Effect.retry({
times: 8,
schedule: retrySchedule<machines.CreateMachineLeaseError>(),
while: (error) => error._tag === "Conflict" || error._tag === "TooManyRequests",
}),
Effect.timeout("30 seconds"),
Effect.catchTag("Conflict", () =>
Effect.fail(new MachineLeaseBusy({ appName, machineId })),
),
Effect.catchTag(["HttpClientError", "GatewayTimeout", "TimeoutError"], (error) =>
lose(machineId, `acquisition uncertain: ${error._tag}`),
),
),
);
const lease = yield* record(machineId, response, requestedAt);
const renewal = yield* heartbeat(machineId, lease).pipe(
Effect.interruptible,
Effect.forkIn(scope),
);
renewals.add(renewal);
}),
);
yield* check;
}
});
const nonce = (machineId: string) =>
Effect.gen(function* () {
yield* check;
const lease = held.get(machineId);
if (!lease) return yield* lose(machineId, "target is outside the acquired lease set");
return lease.nonce;
});
const mutate = <A, E extends { readonly _tag: string }, R>(
machineId: string,
operation: (nonce: string) => Effect.Effect<A, E, R>,
options: { idempotent?: boolean; timeoutMs?: number } = {},
) =>
guard(
Effect.gen(function* () {
const value = yield* nonce(machineId);
return yield* operation(value).pipe(Retry.none);
}).pipe(
Effect.retry({
times: 8,
schedule: retrySchedule<E | MachineLeaseLost>(),
// Only explicit throttling is safe to replay for update/stop/delete.
while: (error) =>
error._tag === "TooManyRequests" ||
(options.idempotent === true &&
(error._tag === "InternalServerError" ||
error._tag === "BadGateway" ||
error._tag === "ServiceUnavailable" ||
error._tag === "GatewayTimeout")),
}),
Effect.timeout(options.timeoutMs ?? 120_000),
Effect.catch((error) =>
Effect.gen(function* () {
if (error._tag === "Conflict" || error._tag === "Forbidden") {
return yield* lose(machineId, `leased mutation rejected: ${error._tag}`);
}
// Transport errors retain the request, including its private nonce header.
if (
error._tag === "HttpClientError" ||
error._tag === "GatewayTimeout" ||
error._tag === "TimeoutError"
) {
return yield* new MachineMutationUncertain({
appName,
machineId,
});
}
return yield* Effect.fail(error);
}),
),
),
);
const forget = (machineId: string) =>
Effect.sync(() => {
held.delete(machineId);
});
const confirmRemoved = (machineId: string) =>
guard(
machines
.getMachine({
app_name: appName,
machine_id: machineId,
})
.pipe(
Retry.none,
Effect.map((machine) => machine.id === machineId && machine.state === "destroyed"),
Effect.catchTag("NotFound", () => Effect.succeed(true)),
Effect.catchTag(
[
"TooManyRequests",
"InternalServerError",
"BadGateway",
"ServiceUnavailable",
"GatewayTimeout",
],
() => Effect.succeed(false),
),
Effect.catchReason("HttpClientError", "TransportError", () => Effect.succeed(false)),
Effect.repeat({
times: 8,
schedule: Schedule.spaced("2 seconds"),
until: (absent) => absent,
}),
Effect.timeoutOrElse({ duration: "30 seconds", orElse: () => Effect.succeed(false) }),
Effect.catch((error) =>
Effect.gen(function* () {
if (!held.has(machineId)) return true;
if (error._tag === "Forbidden") {
return yield* lose(machineId, "removal observation forbidden");
}
return yield* error;
}),
),
// Another observer's confirmed removal wins over a late uncertain response.
Effect.map((absent) => absent || !held.has(machineId)),
Effect.tap((absent) => (absent ? forget(machineId) : Effect.void)),
),
);
const remove = <A, E extends { readonly _tag: string }, R>(
machineId: string,
operation: (nonce: string) => Effect.Effect<A, E, R>,
) =>
Effect.gen(function* () {
const http = yield* HttpClient.HttpClient;
const fetchOptions = yield* Effect.serviceOption(FetchHttpClient.RequestInit);
yield* Effect.sync(() => {
deleting.add(machineId);
});
yield* mutate(machineId, (nonce) =>
operation(nonce).pipe(
// Bun can replay DELETE on a reused socket below the SDK retry policy.
Effect.provideService(
HttpClient.HttpClient,
HttpClient.mapRequest(http, HttpClientRequest.setHeader("connection", "close")),
),
Effect.provideService(FetchHttpClient.RequestInit, {
...Option.getOrUndefined(fetchOptions),
keepalive: false,
redirect: "error",
}),
),
).pipe(
Effect.asVoid,
Effect.catch((error) =>
Effect.gen(function* () {
if (error._tag === "NotFound") return yield* forget(machineId);
if (
error._tag === "Fly.MachineMutationUncertain" ||
error._tag === "InternalServerError" ||
error._tag === "BadGateway" ||
error._tag === "ServiceUnavailable"
) {
// Keep renewal aware of the removal until its exact target has been observed.
if (yield* confirmRemoved(machineId)) return;
}
return yield* Effect.fail(error);
}),
),
);
yield* forget(machineId);
}).pipe(
Effect.ensuring(
Effect.sync(() => {
deleting.delete(machineId);
}),
),
);
const nonceIfHeld = (machineId: string) =>
check.pipe(Effect.map(() => held.get(machineId)?.nonce));
yield* Effect.addFinalizer(() =>
Effect.gen(function* () {
// A late refresh can recreate a lease after a successful release.
yield* Fiber.interruptAll(renewals);
yield* Effect.forEach(
[...held.entries()]
.filter(([machineId]) => !uncertain.has(machineId))
.sort(([a], [b]) => a.localeCompare(b)),
([machineId, lease]) =>
machines
.machinesReleaseLease({
app_name: appName,
machine_id: machineId,
lease_nonce: lease.nonce,
})
.pipe(
Retry.none,
Effect.timeout("5 seconds"),
Effect.interruptible,
Effect.catchTag("NotFound", () => Effect.void),
Effect.catch((error) =>
Effect.logWarning(
"Fly lease release was not confirmed; expiry remains the safety bound",
{ appName, machineId, error: error._tag },
),
),
Effect.asVoid,
),
{ concurrency: 4 },
);
if (uncertain.size > 0)
yield* Effect.logWarning(
"Fly lease renewal was not confirmed; expiry remains the safety bound",
{ appName, machineIds: [...uncertain] },
);
}),
);
const refresh = (machineId: string, lease: HeldLease) =>
Effect.gen(function* () {
yield* check;
if (held.get(machineId) !== lease) return;
const requestedAt = yield* Clock.currentTimeMillis;
const response = yield* machines
.createMachineLease({
app_name: appName,
machine_id: machineId,
ttl: TTL_SECONDS,
lease_nonce: lease.nonce,
description: "Alchemy Machine lifecycle",
})
.pipe(Retry.none);
if (held.get(machineId) !== lease) return;
if (response.data?.nonce !== lease.nonce) {
return yield* lose(machineId, "refresh returned a different nonce");
}
return yield* record(machineId, response, requestedAt);
}).pipe(
Effect.retry({
times: 8,
schedule: retrySchedule<machines.CreateMachineLeaseError | MachineLeaseLost>(),
while: (error) => error._tag === "TooManyRequests",
}),
Effect.timeout("75 seconds"),
Effect.catch((error) =>
Effect.gen(function* () {
if (held.get(machineId) !== lease) return;
uncertain.add(machineId);
// A missing lease during removal is not proof that the Machine is gone.
if (error._tag === "NotFound" && deleting.has(machineId)) {
const absent = yield* confirmRemoved(machineId).pipe(
Effect.catch((observation) =>
held.get(machineId) === lease
? lose(machineId, `removal observation uncertain: ${observation._tag}`)
: Effect.succeed(true),
),
);
if (absent || held.get(machineId) !== lease) return;
return yield* lose(machineId, "lease disappeared before removal was confirmed");
}
return yield* lose(machineId, `refresh uncertain: ${error._tag}`);
}),
),
(operation) =>
Effect.uninterruptible(
Effect.gen(function* () {
const pending = yield* operation.pipe(Effect.interruptible, Effect.forkChild);
return yield* Fiber.join(pending);
}),
),
);
// Each target renews serially on its own cadence within the operation scope.
const heartbeat = (machineId: string, acquired: HeldLease) =>
Effect.gen(function* () {
let lease: HeldLease | undefined = acquired;
while (lease && held.get(machineId) === lease) {
const now = yield* Clock.currentTimeMillis;
yield* Effect.sleep(Math.max(0, lease.renewAt - now));
lease = yield* refresh(machineId, lease);
}
}).pipe(Effect.ignore);
yield* Effect.sleep("1 second").pipe(
Effect.andThen(check),
Effect.forever,
Effect.ignore,
Effect.forkScoped,
);
const checkTarget = (machineId: string) => nonce(machineId).pipe(Effect.asVoid);
return {
acquire,
check,
checkTarget,
guard,
mutate,
remove,
forget,
nonceIfHeld,
};
});
export type MachineLeases = Effect.Success<ReturnType<typeof makeMachineLeases>>;
export const usingMachineLeases = <A, E, R>(
appName: string,
existing: MachineLeases | undefined,
use: (leases: MachineLeases) => Effect.Effect<A, E, R>,
) =>
Effect.gen(function* () {
if (existing) return yield* existing.guard(Effect.suspend(() => use(existing)));
return yield* Effect.scoped(
Effect.gen(function* () {
const leases = yield* makeMachineLeases(appName);
return yield* leases.guard(Effect.suspend(() => use(leases)));
}),
);
});