sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ Kafka/ ServerlessCluster.ts

import * as kafka from "@distilled.cloud/aws/kafka";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import type { Providers } from "../Providers.ts";

export class ServerlessClusterNotReady extends Data.TaggedError("ServerlessClusterNotReady")<{
  readonly message: string;
  readonly cause?: unknown;
}> {}

export class ServerlessClusterUnexpectedResponse extends Data.TaggedError(
  "ServerlessClusterUnexpectedResponse",
)<{
  readonly message: string;
  readonly cause?: unknown;
}> {}

export interface ServerlessClusterProps {
  /**
   * Name of the MSK Serverless cluster. Must be 1-64 characters. If omitted, a
   * deterministic physical name is generated. Changing the name replaces the
   * cluster.
   */
  clusterName?: string;
  /**
   * VPC subnets the cluster's brokers are reachable from. At least two subnets
   * in different Availability Zones are required. Changing subnets replaces the
   * cluster.
   */
  subnetIds: string[];
  /**
   * VPC security groups that control network access to the cluster. If omitted,
   * the VPC's default security group is used. Changing security groups replaces
   * the cluster.
   */
  securityGroupIds?: string[];
  /**
   * User-defined tags for the cluster.
   */
  tags?: Record<string, string>;
}

export interface ServerlessCluster extends Resource<
  "AWS.Kafka.ServerlessCluster",
  ServerlessClusterProps,
  {
    /** Name of the cluster. */
    clusterName: string;
    /** ARN of the cluster. */
    clusterArn: string;
    /** Cluster type — always `SERVERLESS` for this resource. */
    clusterType: string;
    /** Current state (`CREATING`, `ACTIVE`, `DELETING`, ...). */
    state: string;
    /**
     * SASL/IAM bootstrap broker connection string — the endpoint kafkajs-style
     * clients connect to using IAM authentication.
     */
    bootstrapBrokerStringSaslIam: string | undefined;
    /** Subnets the cluster is reachable from. */
    subnetIds: string[];
    /** Security groups attached to the cluster. */
    securityGroupIds: string[];
    /** Tags observed on the cluster. */
    tags: Record<string, string>;
  },
  never,
  Providers
> {}

/**
 * An Amazon MSK (Managed Streaming for Apache Kafka) **Serverless** cluster.
 *
 * MSK Serverless clusters use IAM authentication exclusively and scale broker
 * capacity automatically — there is no broker count, instance type, or storage
 * to configure. They are reachable only from inside a VPC. Creation takes
 * roughly 5-10 minutes.
 *
 * The provisioned (broker-count) MSK cluster is a separate, much slower
 * (~20-40 minute) resource and is intentionally not modeled here.
 *
 * ### Creating a Serverless Cluster
 * **Example:** Serverless Cluster in a VPC
 * ```typescript
 * const cluster = yield* ServerlessCluster("Events", {
 *   subnetIds: [subnetA.subnetId, subnetB.subnetId],
 *   securityGroupIds: [kafkaSecurityGroup.securityGroupId],
 * });
 * ```
 *
 * ### Consuming from a Lambda Function
 * **Example:** Wire a topic to a Lambda
 * ```typescript
 * yield* Kafka.consumeKafkaTopic(cluster, { topics: ["orders"] }, (records) =>
 *   records.pipe(Stream.runForEach((r) => Effect.log(r.value))),
 * );
 * ```
 *
 * @resource
 */
export const ServerlessCluster = Resource<ServerlessCluster>("AWS.Kafka.ServerlessCluster");

const sameStringSet = (
  a: readonly string[] | undefined,
  b: readonly string[] | undefined,
): boolean => {
  const left = [...(a ?? [])].sort();
  const right = [...(b ?? [])].sort();
  return left.length === right.length && left.every((v, i) => v === right[i]);
};

const toTagRecord = (
  tags: { [key: string]: string | undefined } | undefined,
): Record<string, string> =>
  Object.fromEntries(
    Object.entries(tags ?? {}).filter(
      (entry): entry is [string, string] => typeof entry[1] === "string",
    ),
  );

export const ServerlessClusterProvider = () =>
  Provider.effect(
    ServerlessCluster,
    Effect.gen(function* () {
      const toName = (id: string, props: Partial<ServerlessClusterProps>) =>
        props.clusterName
          ? Effect.succeed(props.clusterName)
          : createPhysicalName({ id, maxLength: 64 });

      // Serverless clusters can only be looked up by ARN via describeClusterV2.
      // Before we have an ARN (or after a state-persistence failure) we scan
      // listClustersV2 filtered by name.
      const findByName = Effect.fn(function* (name: string) {
        return yield* kafka.listClustersV2
          .pages({ ClusterNameFilter: name, ClusterTypeFilter: "SERVERLESS" })
          .pipe(
            Stream.runCollect,
            Effect.map((chunk) =>
              Array.from(chunk)
                .flatMap((page) => page.ClusterInfoList ?? [])
                .find((c) => c.ClusterName === name),
            ),
          );
      });

      const describeByArn = Effect.fn(function* (arn: string) {
        const response = yield* kafka
          .describeClusterV2({ ClusterArn: arn })
          .pipe(Effect.catchTag("NotFoundException", () => Effect.undefined));
        return response?.ClusterInfo;
      });

      const readBrokers = Effect.fn(function* (arn: string) {
        return yield* kafka.getBootstrapBrokers({ ClusterArn: arn }).pipe(
          Effect.map((r) => r.BootstrapBrokerStringSaslIam),
          // Brokers are only resolvable once the cluster is ACTIVE.
          Effect.catchTags({
            BadRequestException: () => Effect.undefined,
            ConflictException: () => Effect.undefined,
          }),
        );
      });

      // Bounded readiness wait. MSK Serverless create typically completes in
      // 5-10 minutes; budget ~15 min (60 * 15s).
      const waitForActive = Effect.fn(function* (arn: string) {
        const policy = Schedule.max([Schedule.fixed("15 seconds"), Schedule.recurs(60)]);
        return yield* describeByArn(arn).pipe(
          Effect.flatMap((cluster) => {
            if (!cluster) {
              return Effect.fail(
                new ServerlessClusterNotReady({ message: `MSK cluster '${arn}' not found` }),
              );
            }
            if (cluster.State === "FAILED") {
              return Effect.die(
                new Error(
                  `MSK cluster '${arn}' entered FAILED state: ${cluster.StateInfo?.Message ?? ""}`,
                ),
              );
            }
            if (cluster.State !== "ACTIVE") {
              return Effect.fail(
                new ServerlessClusterNotReady({
                  message: `MSK cluster '${arn}' not active (state: ${cluster.State})`,
                }),
              );
            }
            return Effect.succeed(cluster);
          }),
          Effect.retry({ schedule: policy }),
        );
      });

      const toAttrs = Effect.fn(function* (cluster: kafka.Cluster) {
        if (!cluster.ClusterName || !cluster.ClusterArn) {
          return yield* new ServerlessClusterUnexpectedResponse({
            message: `MSK cluster is missing its name or ARN (state: ${cluster.State})`,
          });
        }
        const bootstrap =
          cluster.State === "ACTIVE" ? yield* readBrokers(cluster.ClusterArn) : undefined;
        const vpcConfigs = cluster.Serverless?.VpcConfigs ?? [];
        return {
          clusterName: cluster.ClusterName,
          clusterArn: cluster.ClusterArn,
          clusterType: cluster.ClusterType ?? "SERVERLESS",
          state: cluster.State ?? "ACTIVE",
          bootstrapBrokerStringSaslIam: bootstrap,
          subnetIds: vpcConfigs.flatMap((v) => v.SubnetIds ?? []),
          securityGroupIds: vpcConfigs.flatMap((v) => v.SecurityGroupIds ?? []),
          tags: toTagRecord(cluster.Tags),
        };
      });

      return {
        stables: ["clusterName", "clusterArn", "clusterType"],

        diff: Effect.fn(function* ({ id, olds, news }) {
          if (!isResolved(news)) return undefined;
          if ((yield* toName(id, olds ?? {})) !== (yield* toName(id, news ?? {}))) {
            return { action: "replace" } as const;
          }
          // VPC configuration is create-only for serverless clusters.
          if (olds?.subnetIds !== undefined && !sameStringSet(news?.subnetIds, olds.subnetIds)) {
            return { action: "replace" } as const;
          }
          if (
            olds?.securityGroupIds !== undefined &&
            !sameStringSet(news?.securityGroupIds, olds.securityGroupIds)
          ) {
            return { action: "replace" } as const;
          }
        }),

        read: Effect.fn(function* ({ id, olds, output }) {
          const cluster = output?.clusterArn
            ? yield* describeByArn(output.clusterArn)
            : yield* findByName(yield* toName(id, olds ?? {}));
          if (!cluster?.ClusterArn) return undefined;
          const attrs = yield* toAttrs(cluster);
          return (yield* hasAlchemyTags(id, attrs.tags)) ? attrs : Unowned(attrs);
        }),

        reconcile: Effect.fn(function* ({ id, news, output, session }) {
          const name = output?.clusterName ?? (yield* toName(id, news));
          const internalTags = yield* createInternalTags(id);
          const desiredTags = { ...internalTags, ...news.tags };

          // 1. Observe — cloud state is authoritative. Prefer the cached ARN,
          //    fall back to a name scan (recovery / adoption).
          let observed = output?.clusterArn
            ? yield* describeByArn(output.clusterArn)
            : yield* findByName(name);

          // 2. Ensure — create if missing; tolerate a concurrent create
          //    (ConflictException) by re-scanning for the cluster by name.
          if (!observed?.ClusterArn) {
            const created = yield* kafka
              .createClusterV2({
                ClusterName: name,
                Tags: desiredTags,
                Serverless: {
                  VpcConfigs: [
                    {
                      SubnetIds: news.subnetIds,
                      SecurityGroupIds: news.securityGroupIds,
                    },
                  ],
                  ClientAuthentication: { Sasl: { Iam: { Enabled: true } } },
                },
              })
              .pipe(
                Effect.map((r) => r.ClusterArn),
                Effect.catchTag("ConflictException", () =>
                  findByName(name).pipe(Effect.map((c) => c?.ClusterArn)),
                ),
              );
            if (!created) {
              return yield* Effect.die(
                new Error(`MSK ServerlessCluster(${id}) could not be created`),
              );
            }
            observed = yield* describeByArn(created);
          }

          if (!observed?.ClusterArn) {
            return yield* Effect.die(
              new Error(`MSK ServerlessCluster(${id}) could not be reconciled`),
            );
          }
          const arn = observed.ClusterArn;

          // Wait for ACTIVE (bounded) so tag sync and broker resolution work.
          observed = yield* waitForActive(arn);

          // 3. Sync tags — diff against OBSERVED cloud tags so adoption
          //    rewrites ownership tags correctly.
          const { removed, upsert } = diffTags(toTagRecord(observed.Tags), desiredTags);
          if (removed.length > 0) {
            yield* kafka.untagResource({ ResourceArn: arn, TagKeys: removed });
          }
          if (upsert.length > 0) {
            yield* kafka.tagResource({
              ResourceArn: arn,
              Tags: Object.fromEntries(upsert.map(({ Key, Value }) => [Key, Value])),
            });
            observed = (yield* describeByArn(arn)) ?? observed;
          }

          // 4. Return fresh attributes.
          yield* session.note(name);
          return yield* toAttrs(observed);
        }),

        delete: Effect.fn(function* ({ output }) {
          yield* kafka.deleteCluster({ ClusterArn: output.clusterArn }).pipe(
            Effect.catchTag("NotFoundException", () => Effect.void),
            // A cluster still CREATING rejects deletion; retry until it can
            // be deleted (or is gone). Bounded.
            Effect.retry({
              while: (e) => e._tag === "BadRequestException",
              schedule: Schedule.max([Schedule.fixed("15 seconds"), Schedule.recurs(40)]),
            }),
            Effect.catchTag("BadRequestException", () => Effect.void),
          );
        }),

        list: () =>
          kafka.listClustersV2.pages({ ClusterTypeFilter: "SERVERLESS" }).pipe(
            Stream.runCollect,
            Effect.map((chunk) =>
              Array.from(chunk).flatMap((page) =>
                (page.ClusterInfoList ?? []).filter(
                  (c) => c.ClusterName !== undefined && c.ClusterArn !== undefined,
                ),
              ),
            ),
            Effect.flatMap(Effect.forEach((cluster) => toAttrs(cluster), { concurrency: 4 })),
          ),
      };
    }),
  );