sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ StateStore/ State.ts

import type { Credentials } from "@distilled.cloud/aws/Credentials";
import type { Region } from "@distilled.cloud/aws/Region";
import * as s3 from "@distilled.cloud/aws/s3";
import { CredentialsStoreLive } from "alchemy/Auth/Credentials";
import { type BucketObjects, makeBucketState } from "alchemy/State/BucketState";
import * as Provisioning from "alchemy/State/Provisioning";
import { State, StateStoreError } from "alchemy/State/State";
import { recordStateStoreInit } from "alchemy/Telemetry/Metrics";
import { globalLogs } from "alchemy/Util/LogRecord";
import * as Effect from "effect/Effect";
import type { HttpClient } from "effect/http/HttpClient";
import * as Layer from "effect/Layer";
import * as LogLevel from "effect/LogLevel";
import * as References from "effect/References";
import * as Schedule from "effect/Schedule";
import * as Scope from "effect/Scope";
import * as Stream from "effect/Stream";
import { AwsAuth } from "../AuthProvider.ts";
import * as AwsCredentials from "../Credentials.ts";
import * as Endpoint from "../Endpoint.ts";
import { AWSEnvironment, providedOrDefault } from "../Environment.ts";
import * as AwsRegion from "../Region.ts";
import { syncBucketEncryption, type BucketEncryption } from "../S3/Bucket.ts";

/** Maximum number of keys S3 accepts in a single DeleteObjects call. */
const DELETE_BATCH_SIZE = 1000;

export interface S3StateOptions {
  /**
   * A store other than the account's default, by logical name: its bucket is
   * `alchemy-state-<name>-<accountId>-<region>-an`. The bucket is created on
   * first use, except in CI, where `alchemy state bootstrap --backend aws
   * --name <name>` provisions it ahead.
   *
   * @default the account's default store, `alchemy-state-<accountId>-<region>-an`
   */
  name?: string;
  /**
   * A full bucket name instead of one derived from {@link S3StateOptions.name},
   * e.g. an existing bucket. Created on first use like a named store's, in the
   * account-regional namespace, so it must follow the
   * `<prefix>-<accountId>-<region>-an` convention.
   *
   * @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/gpbucketnamespaces.html#account-regional-gp-buckets
   */
  bucketName?: string;
  /**
   * Key prefix within the bucket under which all state objects are
   * stored, e.g. `"alchemy"`. A trailing `/` is appended automatically.
   *
   * @default "" (bucket root)
   */
  prefix?: string;
  /**
   * Default encryption enforced on every fresh state-service initialization.
   * Omission restores AES256, no KMS key, bucket keys disabled, and no blocked
   * encryption types. Set `blockedEncryptionTypes: ["SSE-C"]` to block
   * customer-provided keys. Existing encrypted state objects are not rewritten.
   *
   * @default `{ sseAlgorithm: "AES256", bucketKeyEnabled: false, blockedEncryptionTypes: [] }`
   */
  encryption?: BucketEncryption;
}

/** Whether a state bucket exists. */
export const stateBucketExists = (bucket: string) =>
  s3.headBucket({ Bucket: bucket }).pipe(
    Effect.as(true),
    Effect.catchTag(["NotFound", "NoSuchBucket"], () => Effect.succeed(false)),
  );

/** Context required by the distilled S3 operations. */
type S3Deps = Credentials | HttpClient | Region;

/**
 * Run with Debug and Trace records switched off.
 *
 * The AWS client logs every request payload and parsed response at Debug. For
 * this store those are the serialized state objects, whose values include
 * secrets Alchemy generated and keeps, so a Debug floor inherited from where
 * the store is built (the CLI's run log sets one) must not reach them. Floors
 * already stricter than Info are left alone.
 */
export const withoutSdkDebugLogs = Effect.updateService(References.MinimumLogLevel, (level) =>
  LogLevel.isLessThan(level, "Info") ? "Info" : level,
);

/**
 * State store backed by an AWS S3 bucket.
 *
 * Stack state is persisted as JSON objects in an account-regional S3
 * bucket, laid out exactly like the local state store's file tree with
 * the bucket (plus optional `prefix`) taking the place of the
 * `.alchemy/state` directory:
 *
 * ```
 * s3://{bucket}/{prefix}{stack}/{stage}/{fqn}.json
 * s3://{bucket}/{prefix}{stack}/{stage}/__stack_output__.json
 * s3://{bucket}/{prefix}{stack}/{stage}/__lease__.json
 * ```
 *
 * Concurrent deploys of one stack and stage share an `__lease__.json`
 * object written with `If-None-Match` / `If-Match`. The second deploy
 * fails immediately instead of mixing rows. The holder refreshes the
 * lease until the store closes. A killed process stops refreshing, the
 * lease expires, and the next deploy can take it.
 *
 * The bucket is created lazily on the first state operation if it does
 * not already exist — nothing touches AWS credentials at layer
 * construction time.
 *
 *
 * ### Using the S3 State Store
 * Pass `AWS.state({ backend: "s3" })` as the `state` option of a Stack. By
 * default the state is stored in an account-regional bucket named
 * `alchemy-state-{accountId}-{region}-an`.
 *
 * **Example:** Default bucket
 * ```typescript
 * import * as Alchemy from "alchemy";
 * import * as AWS from "@alchemy.run/aws";
 *
 * const Stack = Alchemy.Stack(
 *   "my-stack",
 *   { providers: AWS.providers(), state: AWS.state({ backend: "s3" }) },
 *   Effect.gen(function* () {
 *     // ...
 *   }),
 * );
 * ```
 *
 * **Example:** Custom bucket and key prefix
 * ```typescript
 * const Stack = Alchemy.Stack(
 *   "my-stack",
 *   {
 *     providers: AWS.providers(),
 *     state: AWS.state({
 *       name: "my-company-state",
 *       prefix: "alchemy",
 *       encryption: {
 *         sseAlgorithm: "aws:kms",
 *         kmsMasterKeyId: "alias/alchemy-state",
 *       },
 *     }),
 *   },
 *   Effect.gen(function* () {
 *     // ...
 *   }),
 * );
 * ```
 *
 * ### Supplying the AWS Environment
 * The store resolves its account, region and credentials from the configured
 * profile, CI credentials or the ambient AWS environment. Provide an
 * `AWSEnvironment` to use another credential source; provide the same layer
 * to `AWS.providers()` so the state bucket and every resource share it.
 *
 * **Example:** Deploy-role credentials shared with the providers
 * ```typescript
 * const Stack = Alchemy.Stack(
 *   "my-stack",
 *   {
 *     providers: AWS.providers().pipe(Layer.provide(environment)),
 *     state: AWS.state({ name: "my-company-state" }).pipe(Layer.provide(environment)),
 *   },
 *   Effect.gen(function* () {
 *     // ...
 *   }),
 * );
 * ```
 *
 * ### Managing SSE-C Restrictions
 * **Example:** Block customer-provided encryption keys on the state bucket
 * ```typescript
 * const stateStore = AWS.state({
 *   encryption: {
 *     sseAlgorithm: "AES256",
 *     blockedEncryptionTypes: ["SSE-C"],
 *   },
 * });
 * ```
 *
 * Omitted `blockedEncryptionTypes` is equivalent to `[]`: no encryption types
 * are blocked, so SSE-C writes are permitted via AWS's `NONE` value. Removing
 * an explicit block clears it. Omitting `encryption` restores AES256, no KMS
 * key, disabled bucket keys, and no encryption restrictions. Existing objects
 * are not rewritten.
 */
export const s3State = (options: S3StateOptions = {}) =>
  Layer.effect(
    State,
    Effect.gen(function* () {
      const scope = yield* Effect.scope;
      const context = yield* Effect.context<S3Deps | AWSEnvironment>();

      // Bind the lease to this layer's scope. The cached effect runs on
      // the first state operation, which may sit in a shorter scope.
      const make = Scope.provide(makeS3State(options), scope).pipe(
        recordStateStoreInit,
        // Account-level: filed in the global run log, not the project's.
        globalLogs,
        Effect.orDie,
        Effect.provideContext(context),
      );

      return yield* Effect.cached(make);
    }),
  ).pipe(Layer.provideMerge(stateEnvironment()), Layer.orDie);

/**
 * Construct an S3-backed {@link StateService}.
 *
 * Construction itself never touches AWS — environment resolution and
 * the ensure-bucket-exists check are deferred into a cached Effect
 * that runs once, on the first state operation.
 */
export const makeS3State = (options: S3StateOptions = {}) =>
  Effect.gen(function* () {
    // Captured under `withoutSdkDebugLogs` (below), so every store call runs
    // with the raised floor.
    const context = yield* Effect.context<S3Deps | AWSEnvironment>();

    const toError = (cause: unknown) =>
      new StateStoreError({
        message: cause instanceof Error ? cause.message : `S3 state store error: ${String(cause)}`,
        cause: cause instanceof Error ? cause : undefined,
      });

    // Close over the captured context so every operation is self-contained.
    const run = <A, E>(effect: Effect.Effect<A, E, S3Deps>): Effect.Effect<A, StateStoreError> =>
      effect.pipe(Effect.provideContext(context), Effect.mapError(toError));

    // Anything that touches AWS credentials must NOT run at layer
    // construction time. Resolving the environment (account/region),
    // deriving the bucket name, and ensuring the bucket exists all run on
    // the first state operation; `makeBucketState` caches the result.
    const objects = Effect.gen(function* () {
      const { accountId, region } = yield* AWSEnvironment.current;
      if (options.name !== undefined) yield* Provisioning.storeName(options.name);
      const bucketName =
        options.bucketName ?? createStateBucketName(accountId, region, options.name);
      const ensure = ensureStateBucket(bucketName, region, options);
      // First use creates the bucket (except in CI); every use converges
      // its settings.
      yield* (yield* stateBucketExists(bucketName))
        ? ensure
        : Provisioning.deployMissing(
            {
              backend: "aws-s3",
              name: options.name,
              label: `S3 state bucket '${bucketName}'`,
              configured: options.bucketName !== undefined,
            },
            ensure,
          );
      return s3Objects(bucketName, run);
    }).pipe(Effect.provideContext(context), Effect.mapError(toError));

    return yield* makeBucketState({ id: "s3", label: "S3", prefix: options.prefix, objects });
  }).pipe(withoutSdkDebugLogs);

/** The S3 {@link BucketObjects} of `bucket`, versioned by ETag. */
const s3Objects = (
  bucket: string,
  run: <A, E>(effect: Effect.Effect<A, E, S3Deps>) => Effect.Effect<A, StateStoreError>,
): BucketObjects => ({
  listKeys: (prefix) =>
    run(
      s3.listObjectsV2.pages({ Bucket: bucket, Prefix: prefix }).pipe(
        Stream.flatMap((page) => Stream.fromIterable(page.Contents ?? [])),
        Stream.map((object) => object.Key),
        Stream.filter((key): key is string => key !== undefined),
        Stream.runCollect,
        Effect.map((keys) => Array.from(keys)),
      ),
    ),
  listChildren: (prefix) =>
    run(
      s3.listObjectsV2.pages({ Bucket: bucket, Prefix: prefix, Delimiter: "/" }).pipe(
        Stream.flatMap((page) => Stream.fromIterable(page.CommonPrefixes ?? [])),
        // `{prefix}{name}/` -> `{name}`
        Stream.map((common) => common.Prefix),
        Stream.filter((p): p is string => p !== undefined),
        Stream.map((p) => p.slice(prefix.length, -1)),
        Stream.runCollect,
        Effect.map((names) => Array.from(names)),
      ),
    ),
  read: (key) =>
    run(
      s3.getObject({ Bucket: bucket, Key: key }).pipe(
        Effect.flatMap((result) =>
          result.Body === undefined
            ? Effect.undefined
            : result.Body.pipe(
                Stream.decodeText,
                Stream.mkString,
                Effect.map((text) => ({ text, version: result.ETag ?? "" })),
              ),
        ),
        Effect.catchTag("NoSuchKey", () => Effect.undefined),
      ),
    ),
  write: (key, body, condition) =>
    run(
      s3
        .putObject({
          Bucket: bucket,
          Key: key,
          Body: body,
          ContentType: "application/json",
          ...(condition === undefined
            ? {}
            : "ifAbsent" in condition
              ? { IfNoneMatch: "*" }
              : { IfMatch: condition.ifVersion }),
        })
        .pipe(
          Effect.flatMap((result) =>
            result.ETag === undefined
              ? Effect.fail(
                  new StateStoreError({
                    message: `S3 state object '${key}' was written without an ETag`,
                  }),
                )
              : Effect.succeed<string | undefined>(result.ETag),
          ),
          Effect.retry({
            while: (error) => error._tag === "ConditionalRequestConflict",
            schedule: Schedule.spaced("200 millis"),
            times: 4,
          }),
          // A lost condition: 412, or 409 after retries.
          Effect.catchTag(
            ["PreconditionFailed", "ConditionalRequestConflict"],
            () => Effect.undefined,
          ),
        ),
    ),
  delete: (key, ifVersion) =>
    run(
      s3
        .deleteObject({
          Bucket: bucket,
          Key: key,
          ...(ifVersion === undefined ? {} : { IfMatch: ifVersion }),
        })
        .pipe(Effect.asVoid),
    ),
  deleteMany: (keys) =>
    run(
      Effect.forEach(
        Array.from({ length: Math.ceil(keys.length / DELETE_BATCH_SIZE) }, (_, i) =>
          keys.slice(i * DELETE_BATCH_SIZE, (i + 1) * DELETE_BATCH_SIZE),
        ),
        (batch) =>
          s3.deleteObjects({
            Bucket: bucket,
            Delete: { Objects: batch.map((Key) => ({ Key })), Quiet: true },
          }),
        { discard: true },
      ),
    ),
});

/**
 * Build the default account-regional state bucket name.
 *
 * Account-regional buckets must follow the naming convention:
 *   `<prefix>-<accountId>-<region>-an`
 *
 * @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/gpbucketnamespaces.html#account-regional-gp-buckets
 */
export const createStateBucketName = (accountId: string, region: string, name?: string) =>
  `alchemy-state-${name === undefined ? "" : `${name}-`}${accountId}-${region}-an`.toLowerCase();

/**
 * Observe-then-ensure the state bucket: head it, create it if missing
 * (tolerating create races), and wait for it to become available.
 */
const ensureStateBucket = (bucket: string, region: string, options: S3StateOptions) =>
  Effect.gen(function* () {
    // An absent bucket surfaces as either `NotFound` (the HEAD 404) or
    // `NoSuchBucket` depending on the namespace/path — treat both as "create
    // it". Catching only `NotFound` let a deleted state bucket (e.g. after a
    // nuke) escape as an uncaught `NoSuchBucket` instead of being recreated.
    const exists = yield* s3.headBucket({ Bucket: bucket }).pipe(
      Effect.map(() => true),
      Effect.catchTag(["NotFound", "NoSuchBucket"], () => Effect.succeed(false)),
    );
    if (!exists) {
      yield* Effect.logInfo(`S3 state store: creating bucket ${bucket} in ${region}`);
      yield* s3
        .createBucket({
          Bucket: bucket,
          // account-regional namespace: the bucket name only needs to be
          // unique within this account+region, so deterministic default
          // names can't collide with other AWS customers.
          BucketNamespace: "account-regional",
          // us-east-1 rejects an explicit LocationConstraint
          ...(region === "us-east-1"
            ? {}
            : {
                CreateBucketConfiguration: {
                  LocationConstraint: region as s3.BucketLocationConstraint,
                },
              }),
        })
        .pipe(
          // Many callers race to create the shared default state bucket on
          // first use. The loser sees the create already done
          // (`BucketAlreadyOwnedByYou`/`BucketAlreadyExists`) or mid-flight
          // (`OperationAborted`). All mean "someone else is creating it".
          Effect.catchTag(
            ["BucketAlreadyOwnedByYou", "BucketAlreadyExists", "OperationAborted"],
            () => Effect.void,
          ),
        );

      // Wait for the bucket to become available. Under a concurrent create
      // the bucket is briefly not yet head-able and object/configuration ops
      // would race ahead of it.
      yield* s3.headBucket({ Bucket: bucket }).pipe(
        Effect.retry({
          while: (error) => error._tag === "NotFound" || error._tag === "NoSuchBucket",
          schedule: Schedule.max([Schedule.spaced("1 second"), Schedule.recurs(10)]),
        }),
      );
    }

    // Secure defaults are reconciled for BOTH newly-created and existing
    // buckets. This makes AWS.state converge out-of-band drift and explicit
    // custom buckets rather than only securing the greenfield path.
    const desiredVersioning = "Enabled";
    const observedVersioning = yield* s3.getBucketVersioning({
      Bucket: bucket,
    });
    if (observedVersioning.Status !== desiredVersioning) {
      yield* s3.putBucketVersioning({
        Bucket: bucket,
        VersioningConfiguration: { Status: desiredVersioning },
      });
    }

    yield* syncBucketEncryption(bucket, options.encryption);

    const desiredPublicAccess: s3.PublicAccessBlockConfiguration = {
      BlockPublicAcls: true,
      IgnorePublicAcls: true,
      BlockPublicPolicy: true,
      RestrictPublicBuckets: true,
    };
    const observedPublicAccess = yield* s3.getPublicAccessBlock({ Bucket: bucket }).pipe(
      Effect.map((result) => result.PublicAccessBlockConfiguration),
      Effect.catchTag("NoSuchPublicAccessBlockConfiguration", () => Effect.undefined),
    );
    const publicAccessFingerprint = (config: s3.PublicAccessBlockConfiguration | undefined) =>
      JSON.stringify({
        blockAcls: config?.BlockPublicAcls ?? false,
        ignoreAcls: config?.IgnorePublicAcls ?? false,
        blockPolicy: config?.BlockPublicPolicy ?? false,
        restrictBuckets: config?.RestrictPublicBuckets ?? false,
      });
    if (
      publicAccessFingerprint(observedPublicAccess) !== publicAccessFingerprint(desiredPublicAccess)
    ) {
      yield* s3.putPublicAccessBlock({
        Bucket: bucket,
        PublicAccessBlockConfiguration: desiredPublicAccess,
      });
    }

    const desiredOwnership = "BucketOwnerEnforced";
    const observedOwnership = yield* s3.getBucketOwnershipControls({ Bucket: bucket }).pipe(
      Effect.map((result) => result.OwnershipControls?.Rules?.[0]?.ObjectOwnership),
      Effect.catchTag("OwnershipControlsNotFoundError", () => Effect.undefined),
    );
    if (observedOwnership !== desiredOwnership) {
      yield* s3.putBucketOwnershipControls({
        Bucket: bucket,
        OwnershipControls: {
          Rules: [{ ObjectOwnership: desiredOwnership }],
        },
      });
    }
  }).pipe(
    // The whole observe→create→wait sequence races other first-callers of the
    // shared bucket; `OperationAborted` (conflicting create) and a transiently
    // absent bucket (`NoSuchBucket`/`NotFound`) are the faces of that race.
    // Retry the entire sequence so a concurrent create that is still settling
    // converges instead of surfacing as a `StateStoreError`.
    Effect.retry({
      while: (e) =>
        e._tag === "OperationAborted" || e._tag === "NoSuchBucket" || e._tag === "NotFound",
      schedule: Schedule.max([Schedule.spaced("2 seconds"), Schedule.recurs(10)]),
    }),
  );

/** The profile's AWS environment, resolved the way {@link state} resolves it. */
export const stateEnvironment = () =>
  // Fresh per call: these derive from the environment below, and shared
  // (memoized) instances built for another environment in the same run
  // would shadow an `AWSEnvironment` provided to this layer.
  Layer.fresh(AwsRegion.fromEnvironment).pipe(
    Layer.provideMerge(Layer.fresh(AwsCredentials.fromEnvironment)),
    Layer.provideMerge(Layer.fresh(Endpoint.fromEnvironment)),
    Layer.provideMerge(providedOrDefault()),
    Layer.provideMerge(AwsAuth),
    Layer.provideMerge(CredentialsStoreLive),
  );

/**
 * Create the state bucket with its secure defaults, or converge an existing
 * one to them: what first use of {@link state} does. With `kmsKeyId`,
 * objects are encrypted with that KMS key.
 */
export const provisionStateBucket = (
  bucket: string,
  region: string,
  kmsKeyId?: string,
): Effect.Effect<void, StateStoreError, S3Deps> =>
  ensureStateBucket(
    bucket,
    region,
    kmsKeyId === undefined
      ? {}
      : {
          encryption: { sseAlgorithm: "aws:kms", kmsMasterKeyId: kmsKeyId, bucketKeyEnabled: true },
        },
  ).pipe(
    Effect.mapError(
      (cause) =>
        new StateStoreError({
          message: `Could not provision state bucket ${bucket}: ${cause._tag}`,
          cause: cause instanceof Error ? cause : undefined,
        }),
    ),
  );