import * as logs from "@distilled.cloud/aws/cloudwatch-logs";
import * as deadline from "@distilled.cloud/aws/deadline";
import { diffTags } from "alchemy/Tags";
import * as Effect from "effect/Effect";
import * as Redacted from "effect/Redacted";
import * as Schedule from "effect/Schedule";
import * as EffectStream from "effect/Stream";
import { AWSEnvironment } from "../Environment.ts";
/**
* Unwrap distilled `SensitiveString` values (`string | Redacted<string>`)
* to a plain string for drift comparison.
*/
export const asPlain = (
value: string | Redacted.Redacted<string> | undefined,
): string | undefined =>
value === undefined ? undefined : Redacted.isRedacted(value) ? Redacted.value(value) : value;
/**
* Build a Deadline Cloud ARN factory bound to the ambient account/region.
* Paths look like `farm/{farmId}`, `farm/{farmId}/queue/{queueId}`,
* `monitor/{monitorId}`.
*/
export const deadlineArnOf = Effect.gen(function* () {
const { accountId, region } = yield* AWSEnvironment.current;
return (path: string) => `arn:aws:deadline:${region}:${accountId}:${path}`;
});
/**
* Read the observed tags of a Deadline resource. A missing resource (or a
* resource type without tag support) reads as an empty tag set.
*/
export const fetchDeadlineTags = Effect.fn(function* (arn: string) {
const response = yield* deadline
.listTagsForResource({ resourceArn: arn })
.pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
return Object.fromEntries(
Object.entries(response?.tags ?? {}).filter(
(entry): entry is [string, string] => entry[1] !== undefined,
),
);
});
/**
* Converge a Deadline resource's tags to the desired set, diffing against
* the OBSERVED cloud tags (never olds/output).
*/
export const syncDeadlineTags = Effect.fn(function* (arn: string, desired: Record<string, string>) {
const observed = yield* fetchDeadlineTags(arn);
const { removed, upsert } = diffTags(observed, desired);
if (removed.length > 0) {
yield* deadline.untagResource({ resourceArn: arn, tagKeys: removed });
}
if (upsert.length > 0) {
yield* deadline.tagResource({
resourceArn: arn,
tags: Object.fromEntries(upsert.map(({ Key, Value }) => [Key, Value])),
});
}
});
/**
* Deadline auto-creates CloudWatch log groups under
* `/aws/deadline/{farmId}/{queueId}` (queue job/session logs) and
* `/aws/deadline/{farmId}/{fleetId}/...` (fleet worker logs), and neither
* `deleteQueue` nor `deleteFarm` removes them — without this reap every
* deleted farm strands orphaned log groups. Idempotent: a group already
* gone (or never created) is not an error.
*/
export const reapDeadlineLogGroups = Effect.fn(function* (prefix: string) {
const groups = yield* logs.describeLogGroups.pages({ logGroupNamePrefix: prefix }).pipe(
EffectStream.runCollect,
Effect.map((chunk) =>
Array.from(chunk)
.flatMap((page) => page.logGroups ?? [])
.map((group) => group.logGroupName)
.filter((name): name is string => name != null),
),
);
yield* Effect.forEach(
groups,
(logGroupName) =>
logs
.deleteLogGroup({ logGroupName })
.pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void)),
{ concurrency: 4, discard: true },
);
});
/**
* Reap every child resource of a farm so `deleteFarm` can succeed. A normal
* stack destroy deletes children before the farm, so this observes nothing —
* but an orphan sweep (nuke) or a mid-run crash can leave a farm whose
* children were never enumerated, and `deleteFarm` then rejects with
* `ConflictException` ("This farm still contains some storage profiles /
* queues / fleets ...") until the retry budget runs out and the farm leaks.
*
* Order matters: queue-fleet/queue-limit associations must be stopped and
* deleted before their queues/fleets/limits; storage profiles may be
* referenced by queues (`allowedStorageProfileIds`) so they go after the
* queues. Every step is idempotent — a child already gone is not an error.
*/
export const reapFarmChildren = Effect.fn(function* (farmId: string) {
// Every list catches ResourceNotFoundException → [] because the farm
// itself may already be gone.
// 1. Queue-fleet associations — stop scheduling, then delete (the delete
// rejects with ConflictException until the stop settles to STOPPED).
const queueFleetAssociations = yield* deadline.listQueueFleetAssociations.items({ farmId }).pipe(
EffectStream.runCollect,
Effect.map((chunk) => Array.from(chunk)),
Effect.catchTag("ResourceNotFoundException", () =>
Effect.succeed([] as deadline.QueueFleetAssociationSummary[]),
),
);
yield* Effect.forEach(
queueFleetAssociations,
(assoc) =>
Effect.gen(function* () {
if (assoc.status !== "STOPPED") {
yield* deadline
.updateQueueFleetAssociation({
farmId,
queueId: assoc.queueId,
fleetId: assoc.fleetId,
status: "STOP_SCHEDULING_AND_CANCEL_TASKS",
})
.pipe(
// Already gone (the association or its farm).
Effect.catchTag("ResourceNotFoundException", () => Effect.void),
);
}
yield* retryWhileConflict(
deadline.deleteQueueFleetAssociation({
farmId,
queueId: assoc.queueId,
fleetId: assoc.fleetId,
}),
).pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
}),
{ concurrency: 4, discard: true },
);
// 2. Queue-limit associations — same stop-then-delete dance.
const queueLimitAssociations = yield* deadline.listQueueLimitAssociations.items({ farmId }).pipe(
EffectStream.runCollect,
Effect.map((chunk) => Array.from(chunk)),
Effect.catchTag("ResourceNotFoundException", () =>
Effect.succeed([] as deadline.QueueLimitAssociationSummary[]),
),
);
yield* Effect.forEach(
queueLimitAssociations,
(assoc) =>
Effect.gen(function* () {
if (assoc.status !== "STOPPED") {
yield* deadline
.updateQueueLimitAssociation({
farmId,
queueId: assoc.queueId,
limitId: assoc.limitId,
status: "STOP_LIMIT_USAGE_AND_CANCEL_TASKS",
})
.pipe(
// Already gone (the association or its farm).
Effect.catchTag("ResourceNotFoundException", () => Effect.void),
);
}
yield* retryWhileConflict(
deadline.deleteQueueLimitAssociation({
farmId,
queueId: assoc.queueId,
limitId: assoc.limitId,
}),
).pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
}),
{ concurrency: 4, discard: true },
);
// 3. Budgets.
const budgets = yield* deadline.listBudgets.items({ farmId }).pipe(
EffectStream.runCollect,
Effect.map((chunk) => Array.from(chunk)),
Effect.catchTag("ResourceNotFoundException", () =>
Effect.succeed([] as deadline.BudgetSummary[]),
),
);
yield* Effect.forEach(
budgets,
(budget) =>
deadline
.deleteBudget({ farmId, budgetId: budget.budgetId })
.pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void)),
{ concurrency: 4, discard: true },
);
// 4. Queues (deletion is asynchronous; the farm delete below retries
// through the drain window). A queue's own log group is reaped too.
const queues = yield* deadline.listQueues.items({ farmId }).pipe(
EffectStream.runCollect,
Effect.map((chunk) => Array.from(chunk)),
Effect.catchTag("ResourceNotFoundException", () =>
Effect.succeed([] as deadline.QueueSummary[]),
),
);
yield* Effect.forEach(
queues,
(queue) =>
retryWhileConflict(deadline.deleteQueue({ farmId, queueId: queue.queueId })).pipe(
Effect.catchTag("ResourceNotFoundException", () => Effect.void),
),
{ concurrency: 4, discard: true },
);
// 5. Fleets (deletion drains workers asynchronously).
const fleets = yield* deadline.listFleets.items({ farmId }).pipe(
EffectStream.runCollect,
Effect.map((chunk) => Array.from(chunk)),
Effect.catchTag("ResourceNotFoundException", () =>
Effect.succeed([] as deadline.FleetSummary[]),
),
);
yield* Effect.forEach(
fleets,
(fleet) =>
retryWhileConflict(deadline.deleteFleet({ farmId, fleetId: fleet.fleetId })).pipe(
Effect.catchTag("ResourceNotFoundException", () => Effect.void),
),
{ concurrency: 4, discard: true },
);
// 6. Limits (their queue-limit associations were deleted above).
const limits = yield* deadline.listLimits.items({ farmId }).pipe(
EffectStream.runCollect,
Effect.map((chunk) => Array.from(chunk)),
Effect.catchTag("ResourceNotFoundException", () =>
Effect.succeed([] as deadline.LimitSummary[]),
),
);
yield* Effect.forEach(
limits,
// deleteLimit is idempotent server-side: neither ResourceNotFound nor
// Conflict is in its error union — deleting a missing limit succeeds.
(limit) => deadline.deleteLimit({ farmId, limitId: limit.limitId }),
{ concurrency: 4, discard: true },
);
// 7. Storage profiles (after queues — a queue's allowedStorageProfileIds
// reference blocks profile deletion with ConflictException).
const storageProfiles = yield* deadline.listStorageProfiles.items({ farmId }).pipe(
EffectStream.runCollect,
Effect.map((chunk) => Array.from(chunk)),
Effect.catchTag("ResourceNotFoundException", () =>
Effect.succeed([] as deadline.StorageProfileSummary[]),
),
);
yield* Effect.forEach(
storageProfiles,
(profile) =>
retryWhileConflict(
deadline.deleteStorageProfile({
farmId,
storageProfileId: profile.storageProfileId,
}),
).pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void)),
{ concurrency: 4, discard: true },
);
});
// Explicitly-typed retry wrappers — an inline `Effect.retry` in provider
// lifecycle code leaks `Retry.Return`'s conditional type into declaration
// emit and widens the provider layer to `unknown` for every consumer of
// `AWS.providers()`.
/**
* Retry through Deadline `ConflictException` (STATUS_CONFLICT while a
* sub-resource drains or a concurrent mutation settles). Bounded.
*/
export const retryWhileConflict = <A, E extends { readonly _tag: string }, R>(
self: Effect.Effect<A, E, R>,
): Effect.Effect<A, E, R> =>
Effect.retry(self, {
while: (e) => e._tag === "ConflictException",
schedule: Schedule.max([Schedule.spaced("6 seconds"), Schedule.recurs(9)]),
});
/**
* A freshly-created farm rejects sub-resource mutations for a short window:
* "Farm ... infrastructure setup in progress" surfaces as a 429
* `ThrottlingException` or a `ConflictException`, and the farm's internal
* components (e.g. its BudgetTracker) surface as `ResourceNotFoundException`
* before they are provisioned. Bounded retry through the settling window
* (~60s). Genuine throttling is also safely absorbed here.
*/
export const retryWhileFarmSettling = <A, E extends { readonly _tag: string }, R>(
self: Effect.Effect<A, E, R>,
): Effect.Effect<A, E, R> =>
Effect.retry(self, {
while: (e) =>
e._tag === "ConflictException" ||
e._tag === "InternalServerException" ||
e._tag === "ResourceNotFoundException" ||
e._tag === "ThrottlingException",
schedule: Schedule.max([Schedule.spaced("6 seconds"), Schedule.recurs(9)]),
});
/**
* A freshly-created IAM role may not have propagated when Deadline validates
* it — createMonitor surfaces this as AccessDeniedException, createFleet /
* updateFleet as the typed `DeadlineRoleNotAssumable` ("Could not
* sts:AssumeRole ... check its trust policy"). Bounded retry through the
* propagation window (~60s).
*/
export const retryThroughIamPropagation = <A, E extends { readonly _tag: string }, R>(
self: Effect.Effect<A, E, R>,
): Effect.Effect<A, E, R> =>
Effect.retry(self, {
while: (e) => e._tag === "AccessDeniedException" || e._tag === "DeadlineRoleNotAssumable",
schedule: Schedule.max([Schedule.spaced("6 seconds"), Schedule.recurs(9)]),
});