sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ DSQL/ Stream.ts

import * as dsql from "@distilled.cloud/aws/dsql";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as EStream from "effect/Stream";
import type { Providers } from "../Providers.ts";

export class StreamNotReady extends Data.TaggedError("StreamNotReady")<{
  readonly message: string;
  readonly cause?: unknown;
}> {}

export interface StreamProps {
  /**
   * Identifier of the DSQL cluster whose committed row-level changes the
   * stream captures. The cluster must be in `ACTIVE` status when the stream
   * is created (an `IDLE` cluster rejects `CreateStream` with a validation
   * error). Changing the cluster replaces the stream.
   */
  clusterId: string;
  /**
   * ARN of the Amazon Kinesis data stream that receives the change records.
   * Must be in the same account and region as the cluster. Configure the
   * Kinesis stream with `maxRecordSizeInKiB: 10240` — CDC records can
   * approach 10 MiB and an undersized stream becomes `IMPAIRED` with
   * `KINESIS_OVERSIZE_RECORD`. Changing the target replaces the stream.
   */
  kinesisStreamArn: string;
  /**
   * ARN of the IAM role Aurora DSQL assumes to write records into the
   * Kinesis stream. The role's trust policy must allow the
   * `dsql.amazonaws.com` service principal and its permissions policy must
   * grant `kinesis:PutRecord`, `kinesis:PutRecords`,
   * `kinesis:DescribeStreamSummary` and `kinesis:ListShards` on the target
   * stream. Changing the role replaces the stream.
   */
  roleArn: string;
  /**
   * Record ordering guarantee. Only `UNORDERED` is currently supported —
   * consumers deduplicate and order by `source.ts_ns`.
   * @default "UNORDERED"
   */
  ordering?: dsql.StreamOrdering;
  /**
   * Record serialization format.
   * @default "JSON"
   */
  format?: dsql.StreamFormat;
  /**
   * User-defined tags for the stream.
   */
  tags?: Record<string, string>;
}

export interface Stream extends Resource<
  "AWS.DSQL.Stream",
  StreamProps,
  {
    /** Identifier of the source cluster. */
    clusterId: string;
    /** The unique stream identifier assigned by DSQL. */
    streamId: string;
    /** The ARN of the stream. */
    streamArn: string;
    /** The current status of the stream, e.g. `ACTIVE`. */
    status: string;
    /** Record ordering guarantee. */
    ordering: string;
    /** Record serialization format. */
    format: string;
    /** ARN of the target Kinesis data stream. */
    kinesisStreamArn: string;
    /** ARN of the IAM role DSQL assumes to write to Kinesis. */
    roleArn: string;
  },
  never,
  Providers
> {}

/**
 * A change data capture (CDC) stream on an Aurora DSQL cluster — delivers
 * committed row-level changes (Debezium-shaped JSON envelopes) to an Amazon
 * Kinesis data stream.
 *
 * Creation is asynchronous (`CREATING` -> `ACTIVE`, typically one to three
 * minutes); the provider waits for `ACTIVE` (bounded) before returning. A
 * stream has no update operation — every property except tags replaces it.
 *
 * Functions consume the change records through the existing Kinesis event
 * source on the **target** stream; DSQL itself never invokes compute
 * directly.
 *
 * ### Creating a CDC Stream
 * **Example:** Stream Cluster Changes into Kinesis
 * ```typescript
 * const cluster = yield* DSQL.Cluster("AppDb", {});
 * const target = yield* Kinesis.Stream("Changes", {
 *   streamMode: "ON_DEMAND",
 *   maxRecordSizeInKiB: 10240,
 * });
 * const role = yield* IAM.Role("CdcRole", {
 *   assumeRolePolicyDocument: JSON.stringify({
 *     Version: "2012-10-17",
 *     Statement: [
 *       {
 *         Effect: "Allow",
 *         Principal: { Service: "dsql.amazonaws.com" },
 *         Action: "sts:AssumeRole",
 *       },
 *     ],
 *   }),
 *   inlinePolicies: {
 *     kinesis: JSON.stringify({
 *       Version: "2012-10-17",
 *       Statement: [
 *         {
 *           Effect: "Allow",
 *           Action: [
 *             "kinesis:PutRecord",
 *             "kinesis:PutRecords",
 *             "kinesis:DescribeStreamSummary",
 *             "kinesis:ListShards",
 *           ],
 *           Resource: target.streamArn,
 *         },
 *       ],
 *     }),
 *   },
 * });
 * const cdc = yield* DSQL.Stream("Cdc", {
 *   clusterId: cluster.clusterId,
 *   kinesisStreamArn: target.streamArn,
 *   roleArn: role.roleArn,
 * });
 * ```
 *
 * **Example:** Consume Change Records with a Function
 * ```typescript
 * // DSQL delivers into the Kinesis stream; consume it with the
 * // Kinesis event source on the target stream.
 * yield* Kinesis.consume(target, (records) =>
 *   Effect.forEach(records, (record) => handleChange(record)),
 * );
 * ```
 *
 * @resource
 */
export const Stream = Resource<Stream>("AWS.DSQL.Stream");

const defaultOrdering = "UNORDERED";
const defaultFormat = "JSON";

export const StreamProvider = () =>
  Provider.effect(
    Stream,
    Effect.gen(function* () {
      const readStream = Effect.fn(function* (clusterIdentifier: string, streamIdentifier: string) {
        return yield* dsql
          .getStream({ clusterIdentifier, streamIdentifier })
          .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
      });

      // Bounded readiness wait — stream creation typically completes in one
      // to three minutes; budget 5 min (60 x 5s). FAILED is terminal.
      const waitForActive = Effect.fn(function* (
        clusterIdentifier: string,
        streamIdentifier: string,
      ) {
        const policy = Schedule.max([Schedule.fixed("5 seconds"), Schedule.recurs(60)]);
        return yield* readStream(clusterIdentifier, streamIdentifier).pipe(
          Effect.flatMap((stream) => {
            if (stream === undefined) {
              return Effect.fail(
                new StreamNotReady({ message: `DSQL stream '${streamIdentifier}' not found` }),
              );
            }
            if (stream.status === "FAILED") {
              return Effect.die(
                `DSQL stream '${streamIdentifier}' failed to provision` +
                  (stream.statusReason ? ` (${stream.statusReason.error})` : ""),
              );
            }
            if (stream.status !== "ACTIVE") {
              return Effect.fail(
                new StreamNotReady({
                  message: `DSQL stream '${streamIdentifier}' not active (status: ${stream.status})`,
                }),
              );
            }
            return Effect.succeed(stream);
          }),
          Effect.retry({ schedule: policy }),
        );
      });

      const toAttrs = (stream: dsql.GetStreamOutput) => ({
        clusterId: stream.clusterIdentifier,
        streamId: stream.streamIdentifier,
        streamArn: stream.arn,
        status: stream.status,
        ordering: stream.ordering,
        format: stream.format,
        kinesisStreamArn: stream.targetDefinition?.kinesis.streamArn ?? "",
        roleArn: stream.targetDefinition?.kinesis.roleArn ?? "",
      });

      return {
        stables: ["clusterId", "streamId", "streamArn"],

        diff: Effect.fn(function* ({ olds, news }) {
          if (!isResolved(news)) return undefined;
          // There is no UpdateStream — everything except tags replaces.
          if (
            news.clusterId !== olds?.clusterId ||
            news.kinesisStreamArn !== olds?.kinesisStreamArn ||
            news.roleArn !== olds?.roleArn ||
            (news.ordering ?? defaultOrdering) !== (olds?.ordering ?? defaultOrdering) ||
            (news.format ?? defaultFormat) !== (olds?.format ?? defaultFormat)
          ) {
            return { action: "replace" } as const;
          }
        }),

        read: Effect.fn(function* ({ id, output }) {
          if (output?.streamId === undefined) return undefined;
          const stream = yield* readStream(output.clusterId, output.streamId);
          if (stream === undefined || stream.status === "DELETED") {
            return undefined;
          }
          const tags = Object.fromEntries(
            Object.entries(stream.tags ?? {}).filter(
              (entry): entry is [string, string] => typeof entry[1] === "string",
            ),
          );
          const attrs = toAttrs(stream);
          return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
        }),

        reconcile: Effect.fn(function* ({ id, news, output, session }) {
          const props = news!;
          const internalTags = yield* createInternalTags(id);
          const desiredTags = { ...internalTags, ...props.tags };

          // 1. Observe — cloud state is authoritative; output caches the id.
          let observed =
            output?.streamId === undefined
              ? undefined
              : yield* readStream(output.clusterId, output.streamId);

          // 2. Ensure — create if missing. DSQL assigns the identifier.
          let streamId = observed?.streamIdentifier ?? output?.streamId;
          if (observed === undefined) {
            const created = yield* dsql
              .createStream({
                clusterIdentifier: props.clusterId,
                targetDefinition: {
                  kinesis: {
                    streamArn: props.kinesisStreamArn,
                    roleArn: props.roleArn,
                  },
                },
                ordering: props.ordering ?? defaultOrdering,
                format: props.format ?? defaultFormat,
                tags: desiredTags,
              })
              .pipe(
                // Two transient races surface as typed errors here:
                // - ConflictException while another cluster operation is in
                //   flight;
                // - ValidationException "Aurora DSQL can't access the
                //   specified IAM role ..." while a freshly created service
                //   role propagates through IAM.
                Effect.retry({
                  while: (e): boolean =>
                    e._tag === "ConflictException" ||
                    (e._tag === "ValidationException" && e.message.includes("IAM role")),
                  schedule: Schedule.max([Schedule.fixed("5 seconds"), Schedule.recurs(24)]),
                }),
              );
            streamId = created.streamIdentifier;
          }

          // Wait for ACTIVE so consumers observe a delivering stream.
          observed = yield* waitForActive(props.clusterId, streamId!);

          // 3. Sync tags — diff against OBSERVED cloud tags.
          const observedTags = Object.fromEntries(
            Object.entries(observed.tags ?? {}).filter(
              (entry): entry is [string, string] => typeof entry[1] === "string",
            ),
          );
          const { upsert, removed } = diffTags(observedTags, desiredTags);
          if (upsert.length > 0) {
            yield* dsql.tagResource({
              resourceArn: observed.arn,
              tags: Object.fromEntries(upsert.map((t) => [t.Key, t.Value])),
            });
          }
          if (removed.length > 0) {
            yield* dsql.untagResource({
              resourceArn: observed.arn,
              tagKeys: removed,
            });
          }

          yield* session.note(streamId!);
          return toAttrs(observed);
        }),

        delete: Effect.fn(function* ({ output }) {
          yield* dsql
            .deleteStream({
              clusterIdentifier: output.clusterId,
              streamIdentifier: output.streamId,
            })
            .pipe(
              Effect.catchTag("ResourceNotFoundException", () => Effect.void),
              // A stream still CREATING rejects delete with a typed
              // conflict; retry briefly until it settles.
              Effect.retry({
                while: (e): boolean => e._tag === "ConflictException",
                schedule: Schedule.max([Schedule.fixed("5 seconds"), Schedule.recurs(24)]),
              }),
            );
          // Wait until fully gone (bounded) so the parent cluster's delete
          // does not trip over a stream still DELETING.
          yield* readStream(output.clusterId, output.streamId).pipe(
            Effect.flatMap((stream) =>
              stream === undefined || stream.status === "DELETED"
                ? Effect.void
                : Effect.fail(
                    new StreamNotReady({
                      message: `DSQL stream '${output.streamId}' still ${stream.status}`,
                    }),
                  ),
            ),
            Effect.retry({
              while: (e): boolean => e instanceof Error,
              schedule: Schedule.max([Schedule.fixed("5 seconds"), Schedule.recurs(36)]),
            }),
            // Best-effort — deletion continues server-side either way.
            Effect.ignore,
          );
        }),

        list: () =>
          Effect.gen(function* () {
            const clusters = yield* dsql.listClusters.items({}).pipe(
              EStream.runCollect,
              Effect.map((c) => Array.from(c)),
            );
            const nested = yield* Effect.forEach(
              clusters,
              (cluster) =>
                dsql.listStreams.items({ clusterIdentifier: cluster.identifier }).pipe(
                  EStream.runCollect,
                  Effect.map((c) => Array.from(c)),
                  Effect.catchTag("ResourceNotFoundException", () => Effect.succeed([])),
                ),
              { concurrency: 4 },
            );
            return yield* Effect.forEach(
              nested.flat(),
              (summary) =>
                readStream(summary.clusterIdentifier, summary.streamIdentifier).pipe(
                  Effect.map((stream) =>
                    stream === undefined || stream.status === "DELETED"
                      ? undefined
                      : toAttrs(stream),
                  ),
                ),
              { concurrency: 4 },
            ).pipe(
              Effect.map((attrs) =>
                attrs.filter((a): a is NonNullable<typeof a> => a !== undefined),
              ),
            );
          }),
      };
    }),
  );