sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ EMRServerless/ Application.ts

import * as emr from "@distilled.cloud/aws/emr-serverless";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags";
import { toWireMinutes } from "alchemy/Util/Duration";
import type * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Option from "effect/Option";
import * as Stream from "effect/Stream";
import type { Providers } from "../Providers.ts";
import { awaitApplicationCreated, awaitApplicationStopped } from "./internal.ts";

/**
 * The engine an EMR Serverless application runs. Changing the type replaces
 * the application.
 */
export type ApplicationType = "SPARK" | "HIVE";

/**
 * Auto-stop behavior for an EMR Serverless {@link Application}.
 */
export interface AutoStopConfiguration {
  /**
   * Whether the application stops automatically after being idle.
   * @default true
   */
  enabled?: boolean;
  /**
   * Idle time after which the application stops automatically — e.g.
   * `"5 minutes"` or `Duration.minutes(5)`. Sent to AWS as whole minutes.
   * @default "15 minutes"
   */
  idleTimeout?: Duration.Input;
}

/** Convert the alchemy-facing auto-stop config to the wire shape (minutes). */
const toWireAutoStop = (config: AutoStopConfiguration): emr.AutoStopConfig => ({
  enabled: config.enabled,
  idleTimeoutMinutes: toWireMinutes(config.idleTimeout),
});

export interface ApplicationProps {
  /**
   * Name of the application (1-64 characters: letters, digits, `.` `_` `/`
   * `#` `-`). Changing the name replaces the application.
   * @default a generated physical name
   */
  applicationName?: string;
  /**
   * The engine type. Changing the type replaces the application.
   * @default "SPARK"
   */
  type?: ApplicationType;
  /**
   * The Amazon EMR release associated with the application, e.g.
   * `emr-7.9.0`. Updatable in place (the application must be in a `CREATED`
   * or `STOPPED` state; the provider stops a `STARTED` application before
   * applying the update).
   */
  releaseLabel: string;
  /**
   * The CPU architecture of the application.
   * @default "X86_64"
   */
  architecture?: "X86_64" | "ARM64";
  /**
   * Pre-initialized capacity kept warm per worker type (e.g. `Driver` and
   * `Executor` for Spark; `HiveDriver` and `TezTask` for Hive). Warm workers
   * bill while the application is started.
   */
  initialCapacity?: Record<string, emr.InitialCapacityConfig>;
  /**
   * The maximum aggregate vCPU, memory and disk the application may scale to.
   */
  maximumCapacity?: emr.MaximumAllowedResources;
  /**
   * Whether the application starts automatically on job submission.
   * @default enabled
   */
  autoStartConfiguration?: emr.AutoStartConfig;
  /**
   * Whether (and after how long idle) the application stops
   * automatically.
   * @default enabled after 15 idle minutes
   */
  autoStopConfiguration?: AutoStopConfiguration;
  /**
   * VPC connectivity (subnet + security group IDs) for jobs that must reach
   * resources in a VPC. Omit for the default non-VPC connectivity.
   */
  networkConfiguration?: emr.NetworkConfiguration;
  /**
   * Tags to apply to the application. Merged with the internal Alchemy tags.
   */
  tags?: Record<string, string>;
}

export interface Application extends Resource<
  "AWS.EMRServerless.Application",
  ApplicationProps,
  {
    /** The ID of the application. */
    applicationId: string;
    /** The name of the application. */
    applicationName: string;
    /** The ARN of the application. */
    applicationArn: string;
    /** The application type (`SPARK` or `HIVE`). */
    type: string;
    /** The EMR release label the application runs (e.g. `emr-7.5.0`). */
    releaseLabel: string;
    /** The application state (e.g. `CREATED`, `STARTED`, `STOPPED`). */
    state?: string;
  },
  {},
  Providers
> {}

/**
 * An Amazon EMR Serverless application — a serverless Spark or Hive
 * environment that automatically provisions and scales workers per job,
 * with no cluster to manage. An application in the `CREATED` or `STOPPED`
 * state costs nothing; billing only occurs for workers while the application
 * is started (including any pre-initialized `initialCapacity`).
 *
 * ### Creating Applications
 * **Example:** Spark Application
 * ```typescript
 * import * as AWS from "@alchemy.run/aws";
 *
 * const app = yield* AWS.EMRServerless.Application("Spark", {
 *   releaseLabel: "emr-7.9.0",
 * });
 * // app.applicationId is passed to StartJobRun
 * ```
 *
 * **Example:** Hive Application with Auto-Stop Tuning
 * ```typescript
 * const app = yield* AWS.EMRServerless.Application("Hive", {
 *   type: "HIVE",
 *   releaseLabel: "emr-7.9.0",
 *   autoStartConfiguration: { enabled: true },
 *   autoStopConfiguration: { enabled: true, idleTimeout: "5 minutes" },
 * });
 * ```
 *
 * ### Capacity
 * **Example:** Pre-Initialized Capacity for Low-Latency Jobs
 * ```typescript
 * const app = yield* AWS.EMRServerless.Application("Warm", {
 *   releaseLabel: "emr-7.9.0",
 *   initialCapacity: {
 *     Driver: {
 *       workerCount: 1,
 *       workerConfiguration: { cpu: "2 vCPU", memory: "4 GB" },
 *     },
 *     Executor: {
 *       workerCount: 2,
 *       workerConfiguration: { cpu: "2 vCPU", memory: "4 GB" },
 *     },
 *   },
 *   maximumCapacity: { cpu: "16 vCPU", memory: "64 GB" },
 * });
 * ```
 *
 * ### Networking
 * **Example:** VPC-Connected Application
 * ```typescript
 * const app = yield* AWS.EMRServerless.Application("InVpc", {
 *   releaseLabel: "emr-7.9.0",
 *   networkConfiguration: {
 *     subnetIds: [subnet.subnetId],
 *     securityGroupIds: [securityGroup.securityGroupId],
 *   },
 * });
 * ```
 *
 * @resource
 */
export const Application = Resource<Application>("AWS.EMRServerless.Application");

/** Normalize a possibly-undefined structural prop for drift comparison. */
const canonical = (value: unknown): string => (value === undefined ? "" : JSON.stringify(value));

/**
 * Derive a deterministic create token from the resource's instance ID so a
 * retried create after a crashed reconcile never double-provisions.
 */
const createToken = (instanceId: string): string =>
  instanceId.replaceAll(/[^a-zA-Z0-9]/g, "").slice(0, 64) || "alchemy";

export const ApplicationProvider = () =>
  Provider.effect(
    Application,
    Effect.gen(function* () {
      const createName = Effect.fn(function* (
        id: string,
        props: { applicationName?: string | undefined },
      ) {
        return props.applicationName ?? (yield* createPhysicalName({ id, maxLength: 64 }));
      });

      const toAttributes = (application: emr.Application) => ({
        applicationId: application.applicationId,
        applicationName: application.name ?? application.applicationId,
        applicationArn: application.arn,
        type: application.type,
        releaseLabel: application.releaseLabel,
        state: application.state,
      });

      const observeById = Effect.fn(function* (applicationId: string) {
        return yield* emr.getApplication({ applicationId }).pipe(
          Effect.map((response) => response.application),
          Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
        );
      });

      // Applications can only be fetched by id; when the id is unknown (state
      // persistence failed before the first reconcile completed), find the
      // live (non-terminated) application carrying the derived physical name.
      const observeByName = Effect.fn(function* (name: string) {
        const summary = yield* emr.listApplications.items({}).pipe(
          Stream.filter((s) => s.name === name && s.state !== "TERMINATED"),
          Stream.runHead,
          Effect.map(Option.getOrUndefined),
        );
        if (summary === undefined) {
          return undefined;
        }
        return yield* observeById(summary.id);
      });

      const observe = Effect.fn(function* (applicationId: string | undefined, name: string) {
        const byId = applicationId !== undefined ? yield* observeById(applicationId) : undefined;
        if (byId !== undefined && byId.state !== "TERMINATED") {
          return byId;
        }
        return yield* observeByName(name);
      });

      const syncTags = Effect.fn(function* (
        application: emr.Application,
        desired: Record<string, string>,
      ) {
        // Diff against OBSERVED cloud tags (adoption may bring foreign tags).
        const observed = Object.fromEntries(
          Object.entries(application.tags ?? {}).filter(
            (entry): entry is [string, string] => entry[1] !== undefined,
          ),
        );
        const { upsert, removed } = diffTags(observed, desired);
        if (upsert.length > 0) {
          yield* emr.tagResource({
            resourceArn: application.arn,
            tags: Object.fromEntries(upsert.map((t) => [t.Key, t.Value])),
          });
        }
        if (removed.length > 0) {
          yield* emr.untagResource({
            resourceArn: application.arn,
            tagKeys: removed,
          });
        }
      });

      return Application.Provider.of({
        stables: ["applicationId", "applicationName", "applicationArn", "type"],

        list: () =>
          Effect.gen(function* () {
            const pages = yield* emr.listApplications.pages({}).pipe(Stream.runCollect);
            return Array.from(pages)
              .flatMap((page) => page.applications)
              .filter((s) => s.state !== "TERMINATED")
              .map((s) => ({
                applicationId: s.id,
                applicationName: s.name ?? s.id,
                applicationArn: s.arn,
                type: s.type,
                releaseLabel: s.releaseLabel,
                state: s.state,
              }));
          }),

        read: Effect.fn(function* ({ id, olds, output }) {
          const name = output?.applicationName ?? (yield* createName(id, olds ?? {}));
          const application = yield* observe(output?.applicationId, name);
          if (application === undefined) {
            return undefined;
          }
          const attrs = toAttributes(application);
          return (yield* hasAlchemyTags(id, application.tags)) ? attrs : Unowned(attrs);
        }),

        diff: Effect.fn(function* ({ id, news, olds }) {
          if (!isResolved(news)) return undefined;
          const oldName = yield* createName(id, olds);
          const newName = yield* createName(id, news);
          if (oldName !== newName) {
            return { action: "replace" } as const;
          }
          if ((olds.type ?? "SPARK") !== (news.type ?? "SPARK")) {
            return { action: "replace" } as const;
          }
          // releaseLabel/capacity/autoStart/autoStop/network/tags → update
        }),

        reconcile: Effect.fn(function* ({ id, news, output, session, instanceId }) {
          const name = output?.applicationName ?? (yield* createName(id, news));
          const type = news.type ?? "SPARK";
          const internalTags = yield* createInternalTags(id);
          const desiredTags = { ...news.tags, ...internalTags };
          const desiredAutoStop =
            news.autoStopConfiguration !== undefined
              ? toWireAutoStop(news.autoStopConfiguration)
              : undefined;

          // 1. OBSERVE — cloud state is authoritative; output is an id cache
          let application = yield* observe(output?.applicationId, name);

          // 2. ENSURE — create if missing; ~1 min async until CREATED
          if (application === undefined || application.state === "TERMINATED") {
            const created = yield* emr
              .createApplication({
                clientToken: createToken(instanceId),
                name,
                type,
                releaseLabel: news.releaseLabel,
                architecture: news.architecture,
                initialCapacity: news.initialCapacity,
                maximumCapacity: news.maximumCapacity,
                autoStartConfiguration: news.autoStartConfiguration,
                autoStopConfiguration: desiredAutoStop,
                networkConfiguration: news.networkConfiguration,
                tags: desiredTags,
              })
              .pipe(
                // a concurrent reconciler already created it — observe instead
                Effect.catchTag("ConflictException", () => Effect.undefined),
              );
            const applicationId =
              created?.applicationId ?? (yield* observeByName(name))?.applicationId;
            if (applicationId === undefined) {
              return yield* emr.ResourceNotFoundException.make({
                message: `application ${name} not visible after create`,
              });
            }
            yield* session.note(`creating application ${name} (async)...`);
            application = yield* awaitApplicationCreated(applicationId);
          } else if (application.state === "CREATING") {
            application = yield* awaitApplicationCreated(application.applicationId);
          }

          // 3. SYNC — diff each mutable aspect (observed vs desired), apply a
          //    single updateApplication carrying only the drifted fields.
          const drift = {
            releaseLabel:
              news.releaseLabel !== application.releaseLabel ? news.releaseLabel : undefined,
            architecture:
              news.architecture !== undefined && news.architecture !== application.architecture
                ? news.architecture
                : undefined,
            initialCapacity:
              news.initialCapacity !== undefined &&
              canonical(news.initialCapacity) !== canonical(application.initialCapacity)
                ? news.initialCapacity
                : undefined,
            maximumCapacity:
              news.maximumCapacity !== undefined &&
              canonical(news.maximumCapacity) !== canonical(application.maximumCapacity)
                ? news.maximumCapacity
                : undefined,
            autoStartConfiguration:
              news.autoStartConfiguration !== undefined &&
              canonical(news.autoStartConfiguration) !==
                canonical(application.autoStartConfiguration)
                ? news.autoStartConfiguration
                : undefined,
            autoStopConfiguration:
              desiredAutoStop !== undefined &&
              canonical(desiredAutoStop) !== canonical(application.autoStopConfiguration)
                ? desiredAutoStop
                : undefined,
            networkConfiguration:
              news.networkConfiguration !== undefined &&
              canonical(news.networkConfiguration) !== canonical(application.networkConfiguration)
                ? news.networkConfiguration
                : undefined,
          };
          if (Object.values(drift).some((v) => v !== undefined)) {
            // Updates require a CREATED or STOPPED state — stop a started
            // application first (autoStart re-starts it on job submission).
            if (application.state === "STARTED" || application.state === "STARTING") {
              yield* emr.stopApplication({
                applicationId: application.applicationId,
              });
            }
            yield* awaitApplicationStopped(application.applicationId);
            // Fresh token per update: distinct desired states must never be
            // deduplicated against each other.
            const updateToken = yield* Effect.sync(() => crypto.randomUUID());
            const updated = yield* emr.updateApplication({
              applicationId: application.applicationId,
              clientToken: updateToken,
              ...drift,
            });
            application = updated.application;
          }

          // 3b. SYNC TAGS — diff against observed cloud tags
          yield* syncTags(application, desiredTags);

          yield* session.note(application.applicationId);
          return toAttributes(application);
        }),

        delete: Effect.fn(function* ({ output }) {
          const application = yield* emr
            .getApplication({ applicationId: output.applicationId })
            .pipe(
              Effect.map((response) => response.application),
              Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
            );
          if (application === undefined || application.state === "TERMINATED") {
            return;
          }
          // Deletion requires a CREATED or STOPPED state.
          if (application.state === "STARTED" || application.state === "STARTING") {
            yield* emr.stopApplication({
              applicationId: application.applicationId,
            });
          }
          yield* awaitApplicationStopped(application.applicationId).pipe(
            Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
          );
          yield* emr
            .deleteApplication({ applicationId: output.applicationId })
            .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
        }),
      });
    }),
  );