sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ Lambda/ KafkaEventSource.ts

import * as Namespace from "alchemy/Namespace";
import * as Output from "alchemy/Output";
import type lambda from "aws-lambda";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Stream from "effect/Stream";
import {
  KafkaEventSource as MSKKafkaEventSource,
  type KafkaEventSourceProps,
  type KafkaEventSourceService,
  type MSKRecord,
} from "../Kafka/ClusterEventSource.ts";
import type { ServerlessCluster } from "../Kafka/ServerlessCluster.ts";
import { EventSourceMapping } from "./EventSourceMapping.ts";
import * as Lambda from "./Function.ts";

export const isMSKEvent = (event: any): event is lambda.MSKEvent =>
  event?.eventSource === "aws:kafka" &&
  typeof event?.records === "object" &&
  event.records !== null;

// The cluster ARN is `arn:...:cluster/name/uuid`. Topic and consumer-group
// resources live under sibling namespaces (`topic/` and `group/`) scoped to the
// same cluster path; MSK IAM auth grants data-plane access via these ARNs.
const topicArnGlob = (clusterArn: string) => `${clusterArn.replace(":cluster/", ":topic/")}/*`;
const groupArnGlob = (clusterArn: string) => `${clusterArn.replace(":cluster/", ":group/")}/*`;
// `arn:{partition}:kafka:{region}:{account}:cluster/...` →
// `arn:{partition}:ec2:{region}:{account}:`
const ec2ArnPrefix = (clusterArn: string) => {
  const [arn, partition, , region, account] = clusterArn.split(":");
  return `${arn}:${partition}:ec2:${region}:${account}:`;
};

/** @binding */
export const KafkaEventSource = Layer.effect(
  MSKKafkaEventSource,
  Effect.gen(function* () {
    const host = yield* Lambda.eventSourceHost("AWS.Lambda.KafkaEventSource");
    const Mapping = yield* EventSourceMapping;

    return Effect.fn(function* <StreamReq = never, Req = never>(
      cluster: ServerlessCluster,
      props: KafkaEventSourceProps,
      process: (
        stream: Stream.Stream<MSKRecord, never, StreamReq>,
      ) => Effect.Effect<void, never, Req>,
    ) {
      // Deploy-time: grant the IAM actions MSK IAM authentication requires and
      // create the event-source mapping. Skipped once running inside the
      // deployed Function (the global guard). Namespaced under the host so the
      // mapping's logical identity is stable.
      if (!globalThis.__ALCHEMY_RUNTIME__) {
        yield* Namespace.push(
          host.LogicalId,
          Effect.gen(function* () {
            yield* host.bind`Allow(${host}, AWS.Lambda.KafkaEventSource(${cluster}))`({
              policyStatements: [
                {
                  // Cluster-level: connect + describe (Lambda poller + IAM auth).
                  Effect: "Allow",
                  Action: [
                    "kafka-cluster:Connect",
                    "kafka-cluster:DescribeCluster",
                    "kafka-cluster:DescribeClusterDynamicConfiguration",
                    "kafka:DescribeClusterV2",
                    "kafka:GetBootstrapBrokers",
                  ],
                  Resource: [cluster.clusterArn],
                },
                {
                  // Topic-level: read records + describe topics.
                  Effect: "Allow",
                  Action: ["kafka-cluster:DescribeTopic", "kafka-cluster:ReadData"],
                  Resource: [cluster.clusterArn.pipe(Output.map(topicArnGlob))],
                },
                {
                  // Consumer-group-level: join/commit as a consumer group.
                  Effect: "Allow",
                  Action: ["kafka-cluster:DescribeGroup", "kafka-cluster:AlterGroup"],
                  Resource: [cluster.clusterArn.pipe(Output.map(groupArnGlob))],
                },
                {
                  // The poller reaches the cluster through ENIs in its VPC;
                  // Lambda validates these on the execution role when the
                  // mapping is created. EC2 Describe* actions do not support
                  // resource-level permissions, so scope them to the
                  // cluster's region instead.
                  Effect: "Allow",
                  Action: [
                    "ec2:DescribeNetworkInterfaces",
                    "ec2:DescribeSecurityGroups",
                    "ec2:DescribeSubnets",
                    "ec2:DescribeVpcs",
                  ],
                  Resource: ["*"],
                  Condition: {
                    StringEquals: {
                      "aws:RequestedRegion": cluster.clusterArn.pipe(
                        Output.map((arn) => arn.split(":")[3]!),
                      ),
                    },
                  },
                },
                {
                  // ENIs may only be created in the cluster's subnets with
                  // the cluster's security groups.
                  Effect: "Allow",
                  Action: ["ec2:CreateNetworkInterface"],
                  Resource: Output.all(
                    cluster.clusterArn,
                    cluster.subnetIds,
                    cluster.securityGroupIds,
                  ).pipe(
                    Output.map(([clusterArn, subnetIds, securityGroupIds]) => [
                      `${ec2ArnPrefix(clusterArn)}network-interface/*`,
                      ...subnetIds.map((id) => `${ec2ArnPrefix(clusterArn)}subnet/${id}`),
                      ...securityGroupIds.map(
                        (id) => `${ec2ArnPrefix(clusterArn)}security-group/${id}`,
                      ),
                    ]),
                  ),
                },
                {
                  // ENIs may only be deleted in the cluster's subnets. Creating
                  // the mapping makes Lambda check this permission with no ENI
                  // and no subnet in the request, which fails a
                  // `network-interface/*` resource or a plain `ec2:Subnet`
                  // condition ("Cannot delete ENIs on the subnets and security
                  // groups associated with the event source"). `IfExists`
                  // passes that check while every real delete, which always
                  // carries the ENI's subnet, stays in the cluster's subnets.
                  Effect: "Allow",
                  Action: ["ec2:DeleteNetworkInterface"],
                  Resource: ["*"],
                  Condition: {
                    StringEquals: {
                      "aws:RequestedRegion": cluster.clusterArn.pipe(
                        Output.map((arn) => arn.split(":")[3]!),
                      ),
                    },
                    ArnEqualsIfExists: {
                      "ec2:Subnet": Output.all(cluster.clusterArn, cluster.subnetIds).pipe(
                        Output.map(([clusterArn, subnetIds]) =>
                          subnetIds.map((id) => `${ec2ArnPrefix(clusterArn)}subnet/${id}`),
                        ),
                      ),
                    },
                  },
                },
              ],
            });

            yield* Mapping(
              `AWS.Lambda.EventSourceMapping(${host.LogicalId}, ${cluster.LogicalId})`,
              {
                functionName: Lambda.qualifiedNameOf(host),
                eventSourceArn: cluster.clusterArn,
                topics: props.topics,
                amazonManagedKafkaEventSourceConfig: props.consumerGroupId
                  ? { ConsumerGroupId: props.consumerGroupId }
                  : undefined,
                batchSize: props.batchSize,
                maximumBatchingWindow: props.maximumBatchingWindow,
                startingPosition: props.startingPosition ?? "LATEST",
                enabled: props.enabled ?? true,
                filterCriteria: props.filterCriteria,
                provisionedPollerConfig: props.provisionedPollerConfig,
              },
            );
          }),
        );
      }

      yield* host.listen(
        Effect.gen(function* () {
          return (event: any) => {
            if (isMSKEvent(event)) {
              // `event.records` maps `topic-partition` → MSKRecord[]; flatten
              // to a single stream of records.
              const records = Object.values(event.records).flat() as MSKRecord[];
              if (records.length > 0) {
                return process(Stream.fromArray(records)).pipe(Effect.orDie);
              }
            }
          };
        }),
      );
    }) as KafkaEventSourceService;
  }),
);