sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ EMRContainers/ VirtualCluster.ts

import * as emrc from "@distilled.cloud/aws/emr-containers";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags";
import * as Effect from "effect/Effect";
import * as Option from "effect/Option";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import type { Providers } from "../Providers.ts";

export interface VirtualClusterContainerProvider {
  /**
   * The name of the underlying EKS cluster. Changing it replaces the virtual
   * cluster.
   */
  id: string;
  /**
   * The container provider type.
   * @default "EKS"
   */
  type?: "EKS";
  /**
   * The namespace registration for the underlying EKS cluster.
   */
  info?: {
    eksInfo: {
      /**
       * The Kubernetes namespace the virtual cluster maps to. Changing it
       * replaces the virtual cluster.
       */
      namespace?: string;
      /**
       * A node label selector restricting which nodes run the jobs.
       */
      nodeLabel?: string;
    };
  };
}

export interface VirtualClusterProps {
  /**
   * Name of the virtual cluster (1-64 characters: letters, digits, `.` `_`
   * `/` `#` `-`). Changing the name replaces the virtual cluster.
   * @default a generated physical name
   */
  virtualClusterName?: string;
  /**
   * The EKS cluster and namespace this virtual cluster maps to. Virtual
   * clusters are immutable — any change replaces the virtual cluster.
   */
  containerProvider: VirtualClusterContainerProvider;
  /**
   * The ID of an EMR containers security configuration to attach. Changing
   * it replaces the virtual cluster.
   */
  securityConfigurationId?: string;
  /**
   * Tags to apply to the virtual cluster. Merged with the internal Alchemy
   * tags.
   */
  tags?: Record<string, string>;
}

export interface VirtualCluster extends Resource<
  "AWS.EMRContainers.VirtualCluster",
  VirtualClusterProps,
  {
    /** The ID of the virtual cluster. */
    virtualClusterId: string;
    /** The name of the virtual cluster. */
    virtualClusterName: string;
    /** The ARN of the virtual cluster. */
    virtualClusterArn: string;
    /** The name of the EKS cluster backing the virtual cluster. */
    eksClusterName: string;
    /** The virtual cluster state (e.g. `RUNNING`, `TERMINATING`). */
    state?: string;
  },
  {},
  Providers
> {}

/**
 * An Amazon EMR on EKS virtual cluster — a registration of a Kubernetes
 * namespace on an EKS cluster as an EMR job-submission target. Virtual
 * clusters consume no resources themselves (no cost while idle); jobs
 * submitted to the virtual cluster run as pods in the mapped namespace.
 *
 * The underlying EKS cluster must grant Amazon EMR on EKS access to the
 * namespace (via EKS access entries when the cluster's authentication mode
 * includes `API`, or the legacy `aws-auth` ConfigMap). Everything except tags
 * is immutable — changes replace the virtual cluster.
 *
 * ### Creating Virtual Clusters
 * **Example:** Register an EKS Namespace with EMR
 * ```typescript
 * import * as AWS from "@alchemy.run/aws";
 *
 * const cluster = yield* AWS.EKS.Cluster("Cluster", {
 *   roleArn: clusterRole.roleArn,
 *   resourcesVpcConfig: { subnetIds },
 *   accessConfig: { authenticationMode: "API_AND_CONFIG_MAP" },
 * });
 *
 * const virtualCluster = yield* AWS.EMRContainers.VirtualCluster("Spark", {
 *   containerProvider: {
 *     id: cluster.clusterName,
 *     info: { eksInfo: { namespace: "emr" } },
 *   },
 * });
 * // virtualCluster.virtualClusterId is passed to StartJobRun
 * ```
 *
 * @resource
 */
export const VirtualCluster = Resource<VirtualCluster>("AWS.EMRContainers.VirtualCluster");

export const VirtualClusterProvider = () =>
  Provider.effect(
    VirtualCluster,
    Effect.gen(function* () {
      const createName = Effect.fn(function* (
        id: string,
        props: { virtualClusterName?: string | undefined },
      ) {
        return props.virtualClusterName ?? (yield* createPhysicalName({ id, maxLength: 64 }));
      });

      const toContainerProvider = (
        props: VirtualClusterContainerProvider,
      ): emrc.ContainerProvider => ({
        type: props.type ?? "EKS",
        id: props.id,
        info: props.info,
      });

      const toAttributes = (vc: emrc.VirtualCluster) => ({
        virtualClusterId: vc.id!,
        virtualClusterName: vc.name!,
        virtualClusterArn: vc.arn!,
        eksClusterName: vc.containerProvider?.id ?? "",
        state: vc.state,
      });

      const observeById = Effect.fn(function* (id: string) {
        return yield* emrc.describeVirtualCluster({ id }).pipe(
          Effect.map((response) => response.virtualCluster),
          Effect.catchTag(
            ["ResourceNotFoundException", "ValidationException"],
            // a malformed/unknown id is "not present", same as not-found
            () => Effect.undefined,
          ),
        );
      });

      // Terminated virtual clusters linger in list results; only a RUNNING
      // (or transiently ARRESTED) one with our name is "present".
      const live = (state: string | undefined) => state === "RUNNING" || state === "ARRESTED";

      const observeByName = Effect.fn(function* (name: string) {
        return yield* emrc.listVirtualClusters.items({ states: ["RUNNING", "ARRESTED"] }).pipe(
          Stream.filter((vc) => vc.name === name),
          Stream.runHead,
          Effect.map(Option.getOrUndefined),
        );
      });

      const observe = Effect.fn(function* (id: string | undefined, name: string) {
        const byId = id !== undefined ? yield* observeById(id) : undefined;
        if (byId !== undefined && live(byId.state)) {
          return byId;
        }
        return yield* observeByName(name);
      });

      const syncTags = Effect.fn(function* (
        vc: emrc.VirtualCluster,
        desired: Record<string, string>,
      ) {
        // Diff against OBSERVED cloud tags (adoption may bring foreign tags).
        const observed = Object.fromEntries(
          Object.entries(vc.tags ?? {}).filter(
            (entry): entry is [string, string] => entry[1] !== undefined,
          ),
        );
        const { upsert, removed } = diffTags(observed, desired);
        if (upsert.length > 0) {
          yield* emrc.tagResource({
            resourceArn: vc.arn!,
            tags: Object.fromEntries(upsert.map((t) => [t.Key, t.Value])),
          });
        }
        if (removed.length > 0) {
          yield* emrc.untagResource({
            resourceArn: vc.arn!,
            tagKeys: removed,
          });
        }
      });

      return VirtualCluster.Provider.of({
        stables: ["virtualClusterId", "virtualClusterName", "virtualClusterArn", "eksClusterName"],

        list: () =>
          Effect.gen(function* () {
            const pages = yield* emrc.listVirtualClusters
              .pages({ states: ["RUNNING", "ARRESTED"] })
              .pipe(Stream.runCollect);
            return Array.from(pages)
              .flatMap((page) => page.virtualClusters ?? [])
              .filter((vc) => vc.id !== undefined && vc.name !== undefined && vc.arn !== undefined)
              .map(toAttributes);
          }),

        read: Effect.fn(function* ({ id, olds, output }) {
          const name = output?.virtualClusterName ?? (yield* createName(id, olds ?? {}));
          const vc = yield* observe(output?.virtualClusterId, name);
          if (vc?.id === undefined || vc.arn === undefined) {
            return undefined;
          }
          const attrs = toAttributes(vc);
          return (yield* hasAlchemyTags(id, vc.tags)) ? attrs : Unowned(attrs);
        }),

        diff: Effect.fn(function* ({ id, news, olds }) {
          if (!isResolved(news)) return undefined;
          const oldName = yield* createName(id, olds);
          const newName = yield* createName(id, news);
          if (
            oldName !== newName ||
            JSON.stringify(toContainerProvider(olds.containerProvider)) !==
              JSON.stringify(toContainerProvider(news.containerProvider)) ||
            olds.securityConfigurationId !== news.securityConfigurationId
          ) {
            // everything except tags is immutable
            return { action: "replace" } as const;
          }
        }),

        reconcile: Effect.fn(function* ({ id, news, output, session, instanceId }) {
          const name = output?.virtualClusterName ?? (yield* createName(id, news));
          const internalTags = yield* createInternalTags(id);
          const desiredTags = { ...news.tags, ...internalTags };

          // 1. OBSERVE — cloud state is authoritative; output is an id cache
          let vc = yield* observe(output?.virtualClusterId, name);

          // 2. ENSURE — create if missing (synchronous; RUNNING immediately)
          if (vc === undefined) {
            const created = yield* emrc
              .createVirtualCluster({
                // deterministic per instance: a retried create after a crashed
                // reconcile never double-provisions
                clientToken: instanceId.replaceAll(/[^a-zA-Z0-9]/g, "").slice(0, 64) || "alchemy",
                name,
                containerProvider: toContainerProvider(news.containerProvider),
                securityConfigurationId: news.securityConfigurationId,
                tags: desiredTags,
              })
              .pipe(
                // A just-created EKS cluster rejects attachment while its
                // connection (access configuration) is still being updated.
                Effect.retry({
                  while: (e) => e._tag === "EksClusterConnectionUpdating",
                  schedule: Schedule.spaced("10 seconds"),
                  times: 18,
                }),
              );
            vc =
              created.id !== undefined
                ? yield* emrc
                    .describeVirtualCluster({ id: created.id })
                    .pipe(Effect.map((r) => r.virtualCluster))
                : yield* observeByName(name);
            if (vc?.id === undefined) {
              return yield* emrc.ResourceNotFoundException.make({
                message: `virtual cluster ${name} not visible after create`,
              });
            }
          }

          // 3. SYNC TAGS — the only mutable aspect; diff observed vs desired
          yield* syncTags(vc, desiredTags);

          yield* session.note(vc.id!);
          return toAttributes(vc);
        }),

        delete: Effect.fn(function* ({ output }) {
          // deleteVirtualCluster is idempotent-by-observation: a terminated
          // or missing virtual cluster reads as "not live" and is skipped
          // (repeat deletes of a terminated id fail with ValidationException).
          const vc = yield* emrc.describeVirtualCluster({ id: output.virtualClusterId }).pipe(
            Effect.map((response) => response.virtualCluster),
            Effect.catchTag(
              ["ResourceNotFoundException", "ValidationException"],
              () => Effect.undefined,
            ),
          );
          if (vc === undefined || !live(vc.state)) {
            return;
          }
          yield* emrc
            .deleteVirtualCluster({ id: output.virtualClusterId })
            .pipe(Effect.catchTag("ValidationException", () => Effect.void));
        }),
      });
    }),
  );