sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ Batch/ ComputeEnvironment.ts

import * as batch from "@distilled.cloud/aws/batch";
import * as ec2 from "@distilled.cloud/aws/ec2";
import * as iam from "@distilled.cloud/aws/iam";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, diffTags } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { AWSEnvironment, type AccountID } from "../Environment.ts";
import type { Providers } from "../Providers.ts";
import type { RegionID } from "../Region.ts";
import { pollBatch, retryBatch } from "./internal.ts";

export type ComputeEnvironmentName = string;
export type ComputeEnvironmentArn =
  `arn:aws:batch:${RegionID}:${AccountID}:compute-environment/${ComputeEnvironmentName}`;

/**
 * Raised when networking is left implicit but the account/region has no
 * default VPC to fall back to.
 */
export class NoDefaultVpcError extends Data.TaggedError("NoDefaultVpcError")<{
  readonly message: string;
}> {}

/**
 * Raised when AWS Batch finishes reconciling a compute environment in an
 * unusable state. The status reason is preserved so callers see the actual
 * AWS configuration or dependency failure instead of receiving stale
 * `status: "INVALID"` attributes from a successful deployment.
 */
export class ComputeEnvironmentInvalidError extends Data.TaggedError(
  "ComputeEnvironmentInvalidError",
)<{
  readonly computeEnvironmentName: string;
  readonly status: string;
  readonly statusReason: string | undefined;
  readonly message: string;
}> {}

export interface ComputeEnvironmentProps {
  /**
   * Name of the compute environment. If omitted, a unique name is generated.
   * Up to 128 characters (letters, numbers, hyphens, underscores).
   */
  computeEnvironmentName?: string;
  /**
   * Whether AWS Batch manages the compute capacity. Unmanaged environments
   * are useful when capacity is registered separately, and do not require
   * subnets or security groups.
   * Changing this replaces the compute environment.
   * @default "MANAGED"
   */
  managementType?: "MANAGED" | "UNMANAGED";
  /**
   * Fargate capacity type for the managed compute environment.
   * Changing this replaces the compute environment.
   * @default "FARGATE"
   */
  type?: "FARGATE" | "FARGATE_SPOT";
  /**
   * Maximum number of Fargate vCPUs the environment can scale to.
   * @default 4
   */
  maxvCpus?: number;
  /**
   * Number of externally-managed vCPUs available to an unmanaged compute
   * environment.
   * @default 4
   */
  unmanagedvCpus?: number;
  /**
   * VPC subnets the Fargate tasks run in. If omitted, the default VPC's
   * subnets are used.
   */
  subnets?: string[];
  /**
   * Security groups for the Fargate tasks. If omitted, the default VPC's
   * default security group is used.
   */
  securityGroupIds?: string[];
  /**
   * Whether the compute environment accepts jobs from associated queues.
   * @default "ENABLED"
   */
  state?: "ENABLED" | "DISABLED";
  /**
   * Service role ARN. If omitted, AWS Batch uses (and auto-creates on first
   * use) the `AWSServiceRoleForBatch` service-linked role.
   */
  serviceRole?: string;
  /**
   * User-defined tags to apply to the compute environment.
   */
  tags?: Record<string, string>;
}

export interface ComputeEnvironment extends Resource<
  "AWS.Batch.ComputeEnvironment",
  ComputeEnvironmentProps,
  {
    computeEnvironmentName: ComputeEnvironmentName;
    computeEnvironmentArn: ComputeEnvironmentArn;
    ecsClusterArn: string | undefined;
    managementType: "MANAGED" | "UNMANAGED";
    type: "FARGATE" | "FARGATE_SPOT";
    state: "ENABLED" | "DISABLED";
    status: string;
    maxvCpus: number;
    unmanagedvCpus: number;
    subnets: string[];
    securityGroupIds: string[];
    tags: Record<string, string>;
  },
  never,
  Providers
> {}

/**
 * An AWS Batch managed compute environment backed by Fargate (or Fargate
 * Spot) capacity. Fargate compute environments provision in seconds and
 * require no instance management.
 *
 * ### Creating Compute Environments
 * **Example:** Default Fargate Compute Environment
 * ```typescript
 * // Uses the default VPC's subnets and default security group.
 * const ce = yield* Batch.ComputeEnvironment("JobsCE", {});
 * ```
 *
 * **Example:** Unmanaged Compute Environment
 * ```typescript
 * const ce = yield* Batch.ComputeEnvironment("ExternalCapacity", {
 *   managementType: "UNMANAGED",
 *   unmanagedvCpus: 8,
 * });
 * ```
 *
 * **Example:** Fargate Spot with explicit networking
 * ```typescript
 * const ce = yield* Batch.ComputeEnvironment("SpotCE", {
 *   type: "FARGATE_SPOT",
 *   maxvCpus: 16,
 *   subnets: [subnetA.subnetId, subnetB.subnetId],
 *   securityGroupIds: [sg.groupId],
 * });
 * ```
 *
 * ### Composing the Batch chain
 * **Example:** Compute Environment → Job Queue
 * ```typescript
 * const ce = yield* Batch.ComputeEnvironment("JobsCE", {});
 * const queue = yield* Batch.JobQueue("JobsQueue", {
 *   computeEnvironments: [ce.computeEnvironmentArn],
 * });
 * ```
 *
 * @resource
 */
export const ComputeEnvironment = Resource<ComputeEnvironment>("AWS.Batch.ComputeEnvironment");

const toAttributes = (ce: batch.ComputeEnvironmentDetail, tags: Record<string, string>) => ({
  computeEnvironmentName: ce.computeEnvironmentName!,
  computeEnvironmentArn: ce.computeEnvironmentArn as ComputeEnvironmentArn,
  ecsClusterArn: ce.ecsClusterArn,
  managementType: (ce.type ?? "MANAGED") as "MANAGED" | "UNMANAGED",
  type: (ce.computeResources?.type ?? "FARGATE") as "FARGATE" | "FARGATE_SPOT",
  state: (ce.state ?? "ENABLED") as "ENABLED" | "DISABLED",
  status: ce.status ?? "VALID",
  maxvCpus: ce.computeResources?.maxvCpus ?? 0,
  unmanagedvCpus: ce.unmanagedvCpus ?? 0,
  subnets: [...(ce.computeResources?.subnets ?? [])],
  securityGroupIds: [...(ce.computeResources?.securityGroupIds ?? [])],
  tags,
});

const observedTagsOf = (ce: { tags?: { [key: string]: string | undefined } }) =>
  Object.fromEntries(
    Object.entries(ce.tags ?? {}).filter((e): e is [string, string] => typeof e[1] === "string"),
  );

/**
 * IAM eventual-consistency race: a CE validated against a service role (or
 * its policy attachment) created moments earlier in the same deploy can
 * settle INVALID with a CLIENT_ERROR authorization statusReason (e.g.
 * "... is not authorized to perform: ecs:DescribeClusters ...
 * AccessDeniedException") even though the role and policy are correct.
 * Batch only re-validates an INVALID CE on its slow periodic health check
 * (tens of minutes), so waiting on the same CE cannot recover in deploy
 * time — the recovery is to delete the freshly-created CE and re-create.
 */
const isAuthPropagationInvalid = (ce: batch.ComputeEnvironmentDetail | undefined): boolean =>
  ce?.status === "INVALID" &&
  ce.statusReason !== undefined &&
  ce.statusReason.includes("CLIENT_ERROR") &&
  (ce.statusReason.includes("is not authorized to perform") ||
    ce.statusReason.includes("AccessDeniedException") ||
    ce.statusReason.includes("AccessDenied"));

/**
 * Resolve the default VPC's subnets and default security group — used when
 * the user doesn't pin networking explicitly.
 */
const resolveDefaultNetwork = Effect.gen(function* () {
  const vpcs = yield* ec2.describeVpcs({
    Filters: [{ Name: "is-default", Values: ["true"] }],
  });
  const vpcId = vpcs.Vpcs?.find(
    (vpc) => vpc.State === undefined || vpc.State === "available",
  )?.VpcId;
  if (!vpcId) {
    return yield* new NoDefaultVpcError({
      message: "No default VPC found — pass `subnets` and `securityGroupIds` explicitly",
    });
  }
  const subnets = yield* ec2.describeSubnets
    .items({
      Filters: [{ Name: "vpc-id", Values: [vpcId] }],
    })
    .pipe(
      Stream.runCollect,
      Effect.map((chunk) => Array.from(chunk)),
    );
  const groups = yield* ec2.describeSecurityGroups({
    Filters: [
      { Name: "vpc-id", Values: [vpcId] },
      { Name: "group-name", Values: ["default"] },
    ],
  });
  const network = {
    subnets: subnets.flatMap((s) =>
      s.SubnetId && (s.State === undefined || s.State === "available") ? [s.SubnetId] : [],
    ),
    securityGroupIds: (groups.SecurityGroups ?? []).flatMap((g) => (g.GroupId ? [g.GroupId] : [])),
  };
  if (network.subnets.length === 0 || network.securityGroupIds.length === 0) {
    return yield* new NoDefaultVpcError({
      message:
        "Default VPC networking is not ready — pass `subnets` and `securityGroupIds` explicitly",
    });
  }
  return network;
}).pipe(
  Effect.retry({
    while: (error) => error._tag === "NoDefaultVpcError",
    schedule: Schedule.max([Schedule.spaced("3 seconds"), Schedule.recurs(10)]),
  }),
);

export const ComputeEnvironmentProvider = () =>
  Provider.effect(
    ComputeEnvironment,
    Effect.gen(function* () {
      const toName = (id: string, props: ComputeEnvironmentProps = {}) =>
        props.computeEnvironmentName
          ? Effect.succeed(props.computeEnvironmentName)
          : createPhysicalName({ id, maxLength: 128 });

      // A deleted environment lingers as a DELETED record for a while and the
      // name is immediately reusable, so a describe-by-name can briefly
      // return both the tombstone and the live one — prefer the live one.
      const describeOne = (name: string) =>
        batch
          .describeComputeEnvironments({ computeEnvironments: [name] })
          .pipe(
            Effect.map(
              (res) =>
                res.computeEnvironments?.find((ce) => ce.status !== "DELETED") ??
                res.computeEnvironments?.[0],
            ),
          );

      /** Wait until any CREATING/UPDATING/DELETING transition settles. */
      const awaitSettled = (name: string) =>
        pollBatch(
          describeOne(name),
          (ce) =>
            ce === undefined ||
            (ce.status !== "CREATING" && ce.status !== "UPDATING" && ce.status !== "DELETING"),
        );

      const awaitDisabled = (name: string) =>
        pollBatch(
          describeOne(name),
          (ce) =>
            ce === undefined ||
            ce.status === "DELETED" ||
            ce.status === "DELETING" ||
            (ce.state === "DISABLED" && ce.status !== "CREATING" && ce.status !== "UPDATING"),
        );

      const invalid = (name: string, ce: batch.ComputeEnvironmentDetail | undefined) => {
        const status = ce?.status ?? "MISSING";
        const statusReason = ce?.statusReason;
        return new ComputeEnvironmentInvalidError({
          computeEnvironmentName: name,
          status,
          statusReason,
          message: `Compute environment ${name} settled as ${status}${statusReason ? `: ${statusReason}` : ""}`,
        });
      };

      /**
       * Restore the CE's regular (non-service-linked) service role when it no
       * longer exists in IAM, returning the role name to drop afterwards.
       * Batch cannot tear a CE down without assuming this role — a dangling
       * role is the canonical cause of undeletable, forever-INVALID CEs.
       */
      const restoreServiceRoleIfMissing = Effect.fn(function* (serviceRoleArn: string | undefined) {
        if (!serviceRoleArn || serviceRoleArn.includes("/aws-service-role/")) {
          return undefined;
        }
        const roleName = serviceRoleArn.split("/").pop()!;
        const exists = yield* iam.getRole({ RoleName: roleName }).pipe(
          Effect.map(() => true),
          Effect.catchTag("NoSuchEntityException", () => Effect.succeed(false)),
        );
        if (exists) return undefined;
        yield* iam
          .createRole({
            RoleName: roleName,
            AssumeRolePolicyDocument: JSON.stringify({
              Version: "2012-10-17",
              Statement: [
                {
                  Effect: "Allow",
                  Principal: { Service: "batch.amazonaws.com" },
                  Action: "sts:AssumeRole",
                },
              ],
            }),
            Description:
              "Temporarily restored by alchemy so AWS Batch can tear down a compute environment whose service role was deleted",
          })
          .pipe(
            // Concurrent delete of a replaced instance may restore it first.
            Effect.catchTag("EntityAlreadyExistsException", () => Effect.void),
          );
        yield* iam.attachRolePolicy({
          RoleName: roleName,
          PolicyArn: "arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole",
        });
        // IAM propagation before Batch tries to assume the role.
        yield* Effect.sleep("10 seconds");
        return roleName;
      });

      /** Drop a role restored by {@link restoreServiceRoleIfMissing}. */
      const dropRestoredServiceRole = Effect.fn(function* (roleName: string) {
        yield* iam
          .detachRolePolicy({
            RoleName: roleName,
            PolicyArn: "arn:aws:iam::aws:policy/service-role/AWSBatchServiceRole",
          })
          .pipe(Effect.catchTag("NoSuchEntityException", () => Effect.void));
        yield* iam
          .deleteRole({ RoleName: roleName })
          .pipe(Effect.catchTag("NoSuchEntityException", () => Effect.void));
      });

      /**
       * A force nuke cannot rely on stack dependency ordering: a listed
       * compute environment may still be attached to a separately-listed job
       * queue. Delete those blockers first. Ordinary stack deletion never
       * touches out-of-band queues.
       */
      const deleteRelatedJobQueues = (computeEnvironment: string) =>
        Effect.gen(function* () {
          const pages = yield* batch.describeJobQueues.pages({}).pipe(Stream.runCollect);
          const queues = Array.from(pages)
            .flatMap((page) => page.jobQueues ?? [])
            .filter(
              (queue) =>
                queue.jobQueueName &&
                queue.status !== "DELETED" &&
                queue.computeEnvironmentOrder?.some(
                  (entry) =>
                    entry.computeEnvironment === computeEnvironment ||
                    entry.computeEnvironment?.endsWith(
                      `:compute-environment/${computeEnvironment}`,
                    ),
                ),
            );

          yield* Effect.forEach(
            queues,
            (queue) =>
              Effect.gen(function* () {
                const queueName = queue.jobQueueName!;
                const describeQueue = batch
                  .describeJobQueues({ jobQueues: [queueName] })
                  .pipe(
                    Effect.map((response) =>
                      response.jobQueues?.find((candidate) => candidate.status !== "DELETED"),
                    ),
                  );
                let observed = yield* describeQueue;
                if (!observed) return;
                if (
                  observed.status !== "DELETING" &&
                  (observed.state ?? "ENABLED") !== "DISABLED"
                ) {
                  yield* retryBatch(
                    batch.updateJobQueue({
                      jobQueue: queueName,
                      state: "DISABLED",
                    }),
                    (error) => error._tag === "JobQueueBeingModified",
                  ).pipe(Effect.catchTag("JobQueueNotFound", () => Effect.void));
                }
                observed = yield* pollBatch(
                  describeQueue,
                  (candidate) =>
                    candidate === undefined ||
                    candidate.status === "DELETED" ||
                    candidate.status === "DELETING" ||
                    (candidate.state === "DISABLED" &&
                      (candidate.status === "VALID" || candidate.status === "INVALID")),
                );
                if (!observed || observed.status === "DELETED") return;
                if (observed.status !== "DELETING") {
                  yield* retryBatch(
                    batch.deleteJobQueue({ jobQueue: queueName }),
                    (error) => error._tag === "JobQueueBeingModified",
                  ).pipe(Effect.catchTag("JobQueueNotFound", () => Effect.void));
                }
                yield* pollBatch(
                  describeQueue,
                  (candidate) => candidate === undefined || candidate.status === "DELETED",
                );
              }),
            { concurrency: 4, discard: true },
          );
        });

      return {
        stables: ["computeEnvironmentName", "computeEnvironmentArn"],
        diff: Effect.fn(function* ({ id, olds, news }) {
          if (!isResolved(news)) return;
          if ((yield* toName(id, olds ?? {})) !== (yield* toName(id, news ?? {}))) {
            return { action: "replace" } as const;
          }
          if ((olds?.type ?? "FARGATE") !== (news?.type ?? "FARGATE")) {
            return { action: "replace" } as const;
          }
          if ((olds?.managementType ?? "MANAGED") !== (news?.managementType ?? "MANAGED")) {
            return { action: "replace" } as const;
          }
        }),
        read: Effect.fn(function* ({ id, olds, output }) {
          const name = output?.computeEnvironmentName ?? (yield* toName(id, olds ?? {}));
          const ce = yield* describeOne(name);
          if (!ce?.computeEnvironmentArn || ce.status === "DELETED") {
            return undefined;
          }
          return toAttributes(ce, observedTagsOf(ce));
        }),
        list: () =>
          Effect.gen(function* () {
            const pages = yield* batch.describeComputeEnvironments
              .pages({})
              .pipe(Stream.runCollect);
            return Array.from(pages)
              .flatMap((page) => page.computeEnvironments ?? [])
              .flatMap((ce) =>
                ce.computeEnvironmentArn && ce.status !== "DELETED"
                  ? [toAttributes(ce, observedTagsOf(ce))]
                  : [],
              );
          }),
        reconcile: Effect.fn(function* ({ id, news = {}, output, session }) {
          const { accountId, region } = yield* AWSEnvironment.current;
          const name = output?.computeEnvironmentName ?? (yield* toName(id, news));
          const arn =
            `arn:aws:batch:${region}:${accountId}:compute-environment/${name}` as ComputeEnvironmentArn;
          const internalTags = yield* createInternalTags(id);
          const desiredTags = { ...internalTags, ...news.tags };
          const desiredState = news.state ?? "ENABLED";
          const desiredManagementType = news.managementType ?? "MANAGED";
          const desiredMaxvCpus = news.maxvCpus ?? 4;
          const desiredUnmanagedvCpus = news.unmanagedvCpus ?? 4;
          // Implicit networking means the *current* default VPC network, not
          // whichever subnet IDs happened to exist at create time. Account
          // cleanup can recreate the default VPC between reconciliations, so
          // resolve the desired network on every pass and repair stale IDs.
          const fallbackNetwork =
            desiredManagementType === "MANAGED" && (!news.subnets || !news.securityGroupIds)
              ? yield* resolveDefaultNetwork
              : undefined;
          let desiredNetwork =
            desiredManagementType === "MANAGED"
              ? {
                  subnets: news.subnets ?? fallbackNetwork!.subnets,
                  securityGroupIds: news.securityGroupIds ?? fallbackNetwork!.securityGroupIds,
                }
              : undefined;
          const sameSet = (a: readonly string[], b: readonly string[]) =>
            a.length === b.length && [...a].sort().join(",") === [...b].sort().join(",");
          const matchesDesired = (candidate: batch.ComputeEnvironmentDetail) =>
            (candidate.type ?? "MANAGED") === desiredManagementType &&
            (candidate.state ?? "ENABLED") === desiredState &&
            (desiredManagementType === "UNMANAGED"
              ? (candidate.unmanagedvCpus ?? 0) === desiredUnmanagedvCpus
              : (candidate.computeResources?.maxvCpus ?? 0) === desiredMaxvCpus &&
                sameSet(candidate.computeResources?.subnets ?? [], desiredNetwork!.subnets) &&
                sameSet(
                  candidate.computeResources?.securityGroupIds ?? [],
                  desiredNetwork!.securityGroupIds,
                ));

          /**
           * INVALID is terminal until the configuration is changed. It is
           * commonly caused by default-VPC subnet/security-group churn during
           * highly concurrent account tests, so waiting on the same observed
           * configuration cannot recover. Re-resolve implicit networking and
           * submit a bounded repair update instead.
           */
          const awaitValid = Effect.fn(function* () {
            let lastRepairNetwork: string | undefined;
            for (let attempt = 0; attempt < 8; attempt++) {
              const settled = yield* awaitSettled(name);
              if (settled?.status === "VALID" && matchesDesired(settled)) {
                return settled;
              }
              if (!settled || settled.status === "DELETED" || attempt === 7) {
                return yield* invalid(name, settled);
              }

              if (
                desiredManagementType === "MANAGED" &&
                (!news.subnets || !news.securityGroupIds)
              ) {
                const refreshed = yield* resolveDefaultNetwork;
                desiredNetwork = {
                  subnets: news.subnets ?? refreshed.subnets,
                  securityGroupIds: news.securityGroupIds ?? refreshed.securityGroupIds,
                };
              }
              const repairNetwork = [
                desiredManagementType,
                "|",
                ...(desiredNetwork
                  ? [
                      ...[...desiredNetwork.subnets].sort(),
                      "|",
                      ...[...desiredNetwork.securityGroupIds].sort(),
                    ]
                  : [desiredUnmanagedvCpus]),
              ].join(",");
              if (lastRepairNetwork === repairNetwork) {
                // EC2 may briefly keep returning a just-deleted default VPC
                // network. Do not spend the repair budget re-submitting the
                // same known-invalid IDs; wait for EC2's replacement view.
                yield* Effect.sleep("5 seconds");
                continue;
              }
              yield* retryBatch(
                batch.updateComputeEnvironment({
                  computeEnvironment: name,
                  state: desiredState,
                  unmanagedvCpus:
                    desiredManagementType === "UNMANAGED" ? desiredUnmanagedvCpus : undefined,
                  computeResources:
                    desiredManagementType === "MANAGED"
                      ? {
                          maxvCpus: desiredMaxvCpus,
                          subnets: desiredNetwork!.subnets,
                          securityGroupIds: desiredNetwork!.securityGroupIds,
                        }
                      : undefined,
                }),
                (error) => error._tag === "ComputeEnvironmentBeingModified",
              );
              lastRepairNetwork = repairNetwork;
              // Describe can briefly return the pre-update record before the
              // transition becomes visible.
              yield* Effect.sleep("3 seconds");
            }
            return yield* invalid(name, undefined);
          });

          // Observe — cloud state is authoritative.
          let ce = yield* describeOne(name);
          if (ce?.status === "DELETED") ce = undefined;

          // Ensure — create if missing, then wait until the environment
          // settles to VALID (Fargate CEs settle in seconds).
          const create = Effect.asVoid(
            batch.createComputeEnvironment({
              computeEnvironmentName: name,
              type: desiredManagementType,
              state: desiredState,
              unmanagedvCpus:
                desiredManagementType === "UNMANAGED" ? desiredUnmanagedvCpus : undefined,
              computeResources:
                desiredManagementType === "MANAGED"
                  ? {
                      type: news.type ?? "FARGATE",
                      maxvCpus: desiredMaxvCpus,
                      subnets: desiredNetwork!.subnets,
                      securityGroupIds: desiredNetwork!.securityGroupIds,
                    }
                  : undefined,
              serviceRole: news.serviceRole,
              tags: desiredTags,
            }),
          );

          /**
           * Tear down a CE we created in *this* reconcile that settled
           * INVALID from an IAM propagation race (see
           * {@link isAuthPropagationInvalid}). The CE carries no user state
           * yet, so delete-and-recreate is safe. Mirrors the delete
           * lifecycle's recovery: the async teardown itself can fail once on
           * the same not-yet-propagated role, in which case the CE re-settles
           * INVALID and the delete is re-issued.
           */
          const reapInvalidCreate = Effect.fn(function* () {
            yield* retryBatch(
              batch.updateComputeEnvironment({
                computeEnvironment: name,
                state: "DISABLED",
              }),
              (e) => e._tag === "ComputeEnvironmentBeingModified",
            ).pipe(Effect.catchTag("ComputeEnvironmentNotFound", () => Effect.void));
            const disabled = yield* awaitDisabled(name);
            if (!disabled || disabled.status === "DELETED") return;
            const requestDelete = retryBatch(
              batch.deleteComputeEnvironment({ computeEnvironment: name }),
              (e) =>
                e._tag === "ComputeEnvironmentInUse" ||
                e._tag === "ComputeEnvironmentBeingModified",
            ).pipe(Effect.catchTag("ComputeEnvironmentNotFound", () => Effect.void));
            if (disabled.status !== "DELETING") {
              yield* requestDelete;
              // Describe can briefly return the pre-delete record before the
              // DELETING transition becomes visible.
              yield* Effect.sleep("3 seconds");
            }
            const awaitGone = pollBatch(
              describeOne(name),
              (c) => c === undefined || c.status === "DELETED" || c.status === "INVALID",
            );
            let final = yield* awaitGone;
            if (final && final.status === "INVALID") {
              // The async deletion attempt failed (likely the same IAM
              // propagation lag) — re-issue once and wait again.
              yield* requestDelete;
              yield* Effect.sleep("3 seconds");
              final = yield* awaitGone;
            }
            if (final && final.status !== "DELETED") {
              return yield* invalid(name, final);
            }
          });

          /**
           * Create and wait for the CE to settle, recovering (bounded) from
           * the freshly-created-INVALID IAM propagation race by deleting the
           * INVALID CE and re-creating. Each delete cycle takes ~1-2 minutes,
           * which is itself ample IAM propagation time. A CE that settles
           * INVALID for any other reason — or is still auth-INVALID after 3
           * total create attempts — is returned as-is so the caller's
           * awaitValid path fails loudly with the typed
           * ComputeEnvironmentInvalidError.
           */
          const createAndSettle = Effect.gen(function* () {
            yield* create;
            let settled = yield* awaitSettled(name);
            for (let attempt = 0; attempt < 2 && isAuthPropagationInvalid(settled); attempt++) {
              yield* reapInvalidCreate();
              // Small grace beyond the deletion window before re-validating.
              yield* Effect.sleep("5 seconds");
              yield* create;
              settled = yield* awaitSettled(name);
            }
            return settled;
          });

          if (ce?.computeEnvironmentArn && ce.status === "DELETING") {
            // An interrupted destroy left the environment mid-deletion — let
            // it finish, then fall through to a fresh create below.
            ce = yield* awaitSettled(name);
            if (ce?.status === "DELETED") ce = undefined;
          }
          if (!ce?.computeEnvironmentArn) {
            ce = yield* createAndSettle;
          } else if (ce.status !== "VALID" && ce.status !== "INVALID") {
            // A prior modification is still settling — wait before diffing.
            ce = yield* awaitSettled(name);
          }
          if (ce?.status === "DELETED") {
            // The settle above can still land on a tombstone (deletion that
            // completed between observe and settle) — one more create pass.
            ce = yield* createAndSettle;
          }

          if (!ce?.computeEnvironmentArn) {
            return yield* Effect.die(new Error(`ComputeEnvironment ${name} did not settle`));
          }

          // Sync — diff observed against desired, apply only the delta.
          const update: batch.UpdateComputeEnvironmentRequest = {
            computeEnvironment: name,
          };
          let dirty = false;
          if ((ce.state ?? "ENABLED") !== desiredState) {
            update.state = desiredState;
            dirty = true;
          }
          if (desiredManagementType === "UNMANAGED") {
            if ((ce.unmanagedvCpus ?? 0) !== desiredUnmanagedvCpus) {
              update.unmanagedvCpus = desiredUnmanagedvCpus;
              dirty = true;
            }
          } else {
            const resources: batch.ComputeResourceUpdate = {};
            if ((ce.computeResources?.maxvCpus ?? 0) !== desiredMaxvCpus) {
              resources.maxvCpus = desiredMaxvCpus;
            }
            if (!sameSet(ce.computeResources?.subnets ?? [], desiredNetwork!.subnets)) {
              resources.subnets = desiredNetwork!.subnets;
            }
            if (
              !sameSet(
                ce.computeResources?.securityGroupIds ?? [],
                desiredNetwork!.securityGroupIds,
              )
            ) {
              resources.securityGroupIds = desiredNetwork!.securityGroupIds;
            }
            if (Object.keys(resources).length > 0) {
              update.computeResources = resources;
              dirty = true;
            }
          }
          if (dirty) {
            yield* retryBatch(
              batch.updateComputeEnvironment(update),
              (e) => e._tag === "ComputeEnvironmentBeingModified",
            );
            ce = yield* awaitValid();
          } else if (ce.status !== "VALID") {
            ce = yield* awaitValid();
          }

          // Sync tags — diff against OBSERVED cloud tags.
          const { upsert, removed } = diffTags(observedTagsOf(ce), desiredTags);
          if (upsert.length > 0) {
            yield* batch.tagResource({
              resourceArn: arn,
              tags: Object.fromEntries(upsert.map((t) => [t.Key, t.Value])),
            });
          }
          if (removed.length > 0) {
            yield* batch.untagResource({ resourceArn: arn, tagKeys: removed });
          }

          yield* session.note(arn);
          return toAttributes(
            { ...ce, computeEnvironmentName: name, computeEnvironmentArn: arn },
            desiredTags,
          );
        }),
        delete: Effect.fn(function* ({ output, force }) {
          const name = output.computeEnvironmentName;
          const existing = yield* describeOne(name);
          if (!existing || existing.status === "DELETED") return;

          if (force === true) {
            yield* deleteRelatedJobQueues(name);
          }

          // Must be DISABLED before deletion.
          if (existing.status !== "DELETING" && (existing.state ?? "ENABLED") !== "DISABLED") {
            yield* retryBatch(
              batch.updateComputeEnvironment({
                computeEnvironment: name,
                state: "DISABLED",
              }),
              (e) => e._tag === "ComputeEnvironmentBeingModified",
            ).pipe(Effect.catchTag("ComputeEnvironmentNotFound", () => Effect.void));
          }
          const disabled = yield* awaitDisabled(name);
          if (!disabled || disabled.status === "DELETED") return;

          // Batch tears the CE down (its ECS cluster, ENIs, …) by assuming the
          // CE's service role. If that role was deleted first (crashed destroy,
          // force-nuke ordering), the async deletion fails with an sts:AssumeRole
          // CLIENT_ERROR and the CE is stuck INVALID forever. Restore a minimal
          // same-name role (same ARN — role ARNs carry no unique id) so teardown
          // can proceed, and drop it again once the CE is gone. Service-linked
          // roles are never restored (they're account infrastructure).
          const restoredRole = yield* restoreServiceRoleIfMissing(disabled.serviceRole);

          // deleteComputeEnvironment is idempotent (succeeds when missing) but
          // rejects while a JobQueue association is still tearing down — retry
          // through that window (queue deletion is asynchronous).
          const requestDelete = retryBatch(
            batch.deleteComputeEnvironment({ computeEnvironment: name }),
            (e) =>
              e._tag === "ComputeEnvironmentInUse" || e._tag === "ComputeEnvironmentBeingModified",
            24,
          ).pipe(Effect.catchTag("ComputeEnvironmentNotFound", () => Effect.void));
          if (disabled.status !== "DELETING") {
            yield* requestDelete;
            // Describe can briefly return the pre-delete record (e.g. a stale
            // INVALID status) before the DELETING transition becomes visible.
            yield* Effect.sleep("3 seconds");
          }

          // Wait until it's actually gone (CE deletion takes ~1-2 minutes).
          // Settling on INVALID means the async deletion attempt FAILED (e.g.
          // the service role was dangling until the restore above) — it never
          // self-recovers, so re-issue the delete once and wait again.
          const awaitGone = pollBatch(
            describeOne(name),
            (ce) => ce === undefined || ce.status === "DELETED" || ce.status === "INVALID",
          );
          let final = yield* awaitGone;
          if (final && final.status === "INVALID") {
            yield* requestDelete;
            yield* Effect.sleep("3 seconds");
            final = yield* awaitGone;
          } else if (final && final.status !== "DELETED") {
            // Still DELETING when the poll budget expired — a slow but healthy
            // teardown gets one more poll round before failing loudly.
            final = yield* awaitGone;
          }

          if (restoredRole !== undefined && (final === undefined || final.status === "DELETED")) {
            yield* dropRestoredServiceRole(restoredRole);
          }

          // A poll that merely expires MUST NOT report success — the engine
          // would then delete the CE's dependencies (service role, subnets)
          // while teardown is still consuming them, wedging the CE for good.
          if (final && final.status !== "DELETED") {
            return yield* invalid(name, final);
          }
        }),
      };
    }),
  );