sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ StateStore/ State.ts

import * as crypto from "node:crypto";
import type { Credentials } from "@distilled.cloud/gcp/Credentials";
import * as storage from "@distilled.cloud/gcp/storage_v1";
import { CredentialsStoreLive } from "alchemy/Auth/Credentials";
import { type BucketObjects, makeBucketState } from "alchemy/State/BucketState";
import * as Provisioning from "alchemy/State/Provisioning";
import { State, StateStoreError } from "alchemy/State/State";
import { recordStateStoreInit } from "alchemy/Telemetry/Metrics";
import { globalLogs } from "alchemy/Util/LogRecord";
import * as Effect from "effect/Effect";
import type { HttpClient } from "effect/http/HttpClient";
import * as Layer from "effect/Layer";
import * as Schedule from "effect/Schedule";
import * as Scope from "effect/Scope";
import * as Stream from "effect/Stream";
import { GcpAuth } from "../AuthProvider.ts";
import * as GcpCredentials from "../Credentials.ts";
import { fromCredentials, GcpEnvironment } from "../Environment.ts";
import { makeObjectMedia } from "../Storage/ObjectMedia.ts";

export interface GcsStateOptions {
  /**
   * A store other than the project's default, by logical name: its bucket is
   * `alchemy-state-<name>-<project>`. The bucket is created on first use,
   * except in CI, where `alchemy state bootstrap --backend gcp --name <name>`
   * provisions it ahead.
   *
   * @default the project's default store, `alchemy-state-<project>`
   */
  name?: string;
  /**
   * A full bucket name instead of one derived from {@link GcsStateOptions.name},
   * e.g. an existing bucket. Created on first use if it does not exist.
   */
  bucketName?: string;
  /**
   * Key prefix within the bucket under which all state objects are stored,
   * e.g. `"alchemy"`. A trailing `/` is appended automatically.
   *
   * @default "" (bucket root)
   */
  prefix?: string;
  /**
   * Location of a bucket the store creates, e.g. `"us-central1"` or the
   * multi-region `"US"`. An existing bucket keeps its location.
   *
   * @default the profile's region, else `us-central1`
   */
  location?: string;
}

/** Context required by the distilled Cloud Storage operations. */
type GcsDeps = Credentials | HttpClient;

/** Cloud Storage bucket names are at most 63 characters. */
const BUCKET_NAME_MAX = 63;

/**
 * Build the default state bucket name, `alchemy-state-[<name>-]<project>`.
 * Bucket names are global, and project ids are unique, so the name is
 * deterministic per project. A name too long for Cloud Storage replaces the
 * project id with a hash of it.
 */
export const createStateBucketName = (project: string, name?: string) => {
  const base = `alchemy-state-${name === undefined ? "" : `${name}-`}`;
  const full = `${base}${project}`.toLowerCase();
  if (full.length <= BUCKET_NAME_MAX) return full;
  const hash = crypto.createHash("sha256").update(project).digest("hex").slice(0, 16);
  return `${base}${hash}`.toLowerCase();
};

/** Whether a state bucket exists. */
export const stateBucketExists = (bucket: string) =>
  storage.getBuckets({ bucket }).pipe(
    Effect.as(true),
    Effect.catchTag("NotFound", () => Effect.succeed(false)),
  );

/**
 * State store backed by a Google Cloud Storage bucket.
 *
 * Stack state is persisted as JSON objects in a bucket in your project,
 * laid out like the local state store's file tree:
 *
 * ```
 * gs://{bucket}/{prefix}{stack}/{stage}/{fqn}.json
 * gs://{bucket}/{prefix}{stack}/{stage}/__stack_output__.json
 * gs://{bucket}/{prefix}{stack}/{stage}/__lease__.json
 * ```
 *
 * Concurrent deploys of one stack and stage share an `__lease__.json`
 * object written with `ifGenerationMatch` preconditions. The second deploy
 * fails immediately instead of mixing rows.
 *
 * The bucket is created on the first state operation if it does not
 * exist, with uniform bucket-level access, public access prevention, and
 * object versioning. Project Owners and Editors get
 * `roles/storage.objectAdmin` on it, since uniform access otherwise leaves
 * them unable to read objects. Nothing touches GCP credentials at layer
 * construction time.
 *
 * ### Using the Cloud Storage State Store
 * Pass `GCP.state()` as the `state` option of a Stack. By default the state
 * is stored in a bucket named `alchemy-state-{project}`.
 *
 * **Example:** Default bucket
 * ```typescript
 * import * as Alchemy from "alchemy";
 * import * as GCP from "@alchemy.run/gcp";
 * import * as Effect from "effect/Effect";
 *
 * export default Alchemy.Stack(
 *   "MyApp",
 *   { providers: GCP.providers(), state: GCP.state() },
 *   Effect.gen(function* () {
 *     return {};
 *   }),
 * );
 * ```
 *
 * **Example:** Named store with a key prefix
 * ```typescript
 * export default Alchemy.Stack(
 *   "MyApp",
 *   {
 *     providers: GCP.providers(),
 *     state: GCP.state({ name: "platform", prefix: "alchemy" }),
 *   },
 *   Effect.gen(function* () {
 *     return {};
 *   }),
 * );
 * ```
 *
 * ### Provisioning in CI
 * In CI (`CI=true`) a missing bucket fails the run instead of being
 * created. Create it once ahead of time:
 *
 * **Example:** Bootstrap the bucket
 * ```sh
 * alchemy state bootstrap --backend gcp --profile <your-ci-profile>
 * ```
 *
 * @resource
 */
export const state = (options: GcsStateOptions = {}) =>
  Layer.effect(
    State,
    Effect.gen(function* () {
      const scope = yield* Effect.scope;
      const context = yield* Effect.context<GcsDeps | GcpEnvironment>();

      // Bind the lease to this layer's scope. The cached effect runs on the
      // first state operation, which may sit in a shorter scope.
      const make = Scope.provide(makeGcsState(options), scope).pipe(
        recordStateStoreInit,
        // Account-level: filed in the global run log, not the project's.
        globalLogs,
        Effect.orDie,
        Effect.provideContext(context),
      );

      return yield* Effect.cached(make);
    }),
  ).pipe(Layer.provideMerge(stateEnvironment()), Layer.orDie);

/**
 * Construct a Cloud Storage-backed `StateService`. Resolving the project and
 * ensuring the bucket run on the first state operation.
 */
export const makeGcsState = (options: GcsStateOptions = {}) =>
  Effect.gen(function* () {
    const context = yield* Effect.context<GcsDeps | GcpEnvironment>();

    const toError = (cause: unknown) =>
      new StateStoreError({
        message:
          cause instanceof Error
            ? cause.message
            : typeof cause === "object" && cause !== null && "_tag" in cause
              ? `Cloud Storage state store error: ${String(cause._tag)}`
              : `Cloud Storage state store error: ${String(cause)}`,
        cause: cause instanceof Error ? cause : undefined,
      });

    const run = <A, E>(effect: Effect.Effect<A, E, GcsDeps>): Effect.Effect<A, StateStoreError> =>
      effect.pipe(Effect.provideContext(context), Effect.mapError(toError));

    const objects = Effect.gen(function* () {
      const { project, region } = yield* GcpEnvironment.current;
      if (options.name !== undefined) yield* Provisioning.storeName(options.name);
      const bucketName = options.bucketName ?? createStateBucketName(project, options.name);
      const ensure = ensureStateBucket(bucketName, project, options.location ?? region);
      yield* (yield* stateBucketExists(bucketName))
        ? ensure
        : Provisioning.deployMissing(
            {
              backend: "gcp",
              name: options.name,
              label: `Cloud Storage state bucket '${bucketName}'`,
              configured: options.bucketName !== undefined,
            },
            ensure,
          );
      const media = yield* makeObjectMedia;
      return gcsObjects(bucketName, media, run);
    }).pipe(Effect.provideContext(context), Effect.mapError(toError));

    return yield* makeBucketState({
      id: "gcs",
      label: "Cloud Storage",
      prefix: options.prefix,
      objects,
    });
  });

type ObjectMedia = Effect.Success<typeof makeObjectMedia>;

/** The Cloud Storage {@link BucketObjects} of `bucket`, versioned by generation. */
const gcsObjects = (
  bucket: string,
  media: ObjectMedia,
  run: <A, E>(effect: Effect.Effect<A, E, GcsDeps>) => Effect.Effect<A, StateStoreError>,
): BucketObjects => ({
  listKeys: (prefix) =>
    run(
      storage.listObjects.pages({ bucket, prefix }).pipe(
        Stream.flatMap((page) => Stream.fromIterable(page.items ?? [])),
        Stream.map((object) => object.name),
        Stream.filter((name): name is string => name !== undefined),
        Stream.runCollect,
        Effect.map((names) => Array.from(names)),
      ),
    ),
  listChildren: (prefix) =>
    run(
      storage.listObjects.pages({ bucket, prefix, delimiter: "/" }).pipe(
        Stream.flatMap((page) => Stream.fromIterable(page.prefixes ?? [])),
        // `{prefix}{name}/` -> `{name}`
        Stream.map((p) => p.slice(prefix.length, -1)),
        Stream.runCollect,
        Effect.map((names) => Array.from(names)),
      ),
    ),
  read: (key) =>
    run(
      media.download({ bucket, object: key }).pipe(
        Effect.map((object) => ({
          text: new TextDecoder().decode(object.body),
          version: object.generation ?? "",
        })),
        Effect.catchTag("GCP.Storage.ObjectNotFound", () => Effect.undefined),
      ),
    ),
  write: (key, body, condition) =>
    run(
      media
        .upload(bucket, {
          name: key,
          body,
          contentType: "application/json",
          ifGenerationMatch:
            condition === undefined
              ? undefined
              : "ifAbsent" in condition
                ? "0"
                : condition.ifVersion,
        })
        .pipe(
          Effect.map((object): string | undefined => object.generation ?? ""),
          // A lost condition answers 412.
          Effect.catchIf(
            (error) => error._tag === "GCP.Storage.ObjectRequestFailed" && error.status === 412,
            () => Effect.undefined,
          ),
        ),
    ),
  delete: (key, ifVersion) =>
    run(
      storage
        .deleteObjects({
          bucket,
          object: key,
          ...(ifVersion === undefined ? {} : { ifGenerationMatch: ifVersion }),
        })
        .pipe(
          Effect.asVoid,
          Effect.catchTag(["NotFound", "ObjectPreconditionFailed"], () => Effect.void),
        ),
    ),
  deleteMany: (keys) =>
    run(
      Effect.forEach(
        keys,
        (object) =>
          storage.deleteObjects({ bucket, object }).pipe(
            Effect.asVoid,
            Effect.catchTag("NotFound", () => Effect.void),
          ),
        { concurrency: 16, discard: true },
      ),
    ),
});

/** Settings every state bucket converges to. */
const desiredIamConfiguration = {
  uniformBucketLevelAccess: { enabled: true },
  publicAccessPrevention: "enforced",
};

/** The role that lets project owners and editors read and write state. */
const STATE_OBJECT_ROLE = "roles/storage.objectAdmin";

/**
 * With uniform bucket-level access, project Owners and Editors only hold
 * `storage.legacyBucketOwner`/`legacyBucketWriter`, which can list and
 * write objects but not read them. Grant them object access on the state
 * bucket so anyone who can deploy to the project can use the store.
 */
const syncStateBucketAccess = (bucket: string, project: string) =>
  Effect.gen(function* () {
    const members = [`projectOwner:${project}`, `projectEditor:${project}`];
    const policy = yield* storage.getIamPolicyBuckets({ bucket });
    const bindings = policy.bindings ?? [];
    const existing = bindings.find(
      (binding) => binding.role === STATE_OBJECT_ROLE && binding.condition === undefined,
    );
    const missing = members.filter((member) => !(existing?.members ?? []).includes(member));
    if (missing.length === 0) return;
    yield* storage.setIamPolicyBuckets({
      bucket,
      body: {
        ...policy,
        bindings: existing
          ? bindings.map((binding) =>
              binding === existing
                ? { ...binding, members: [...(binding.members ?? []), ...missing] }
                : binding,
            )
          : [...bindings, { role: STATE_OBJECT_ROLE, members }],
      },
    });
  }).pipe(
    Effect.retry({
      // A concurrent policy write, or a just-created bucket not yet visible.
      while: (error) =>
        error._tag === "IamPolicyEtagMismatch" ||
        error._tag === "Conflict" ||
        error._tag === "NotFound",
      schedule: Schedule.spaced("1 second"),
      times: 8,
    }),
  );

/**
 * Observe-then-ensure the state bucket: create it if missing (tolerating
 * create races), then converge uniform access, public access prevention,
 * versioning, and object access for project owners and editors.
 */
const ensureStateBucket = (bucket: string, project: string, location: string) =>
  Effect.gen(function* () {
    const observed = yield* storage
      .getBuckets({ bucket })
      .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
    if (observed === undefined) {
      yield* Effect.logInfo(`Cloud Storage state store: creating bucket ${bucket} in ${location}`);
      yield* storage
        .insertBuckets({
          project,
          body: {
            name: bucket,
            location,
            iamConfiguration: desiredIamConfiguration,
            versioning: { enabled: true },
          },
        })
        .pipe(
          // A concurrent first use created it first.
          Effect.catchTag("Conflict", () => Effect.void),
        );
    } else {
      const drifted =
        observed.iamConfiguration?.uniformBucketLevelAccess?.enabled !== true ||
        observed.iamConfiguration?.publicAccessPrevention !== "enforced" ||
        observed.versioning?.enabled !== true;
      if (drifted) {
        yield* storage.patchBuckets({
          bucket,
          body: {
            iamConfiguration: desiredIamConfiguration,
            versioning: { enabled: true },
          },
        });
      }
    }
    yield* syncStateBucketAccess(bucket, project);
  }).pipe(
    Effect.retry({
      while: (error) => error._tag === "Conflict",
      schedule: Schedule.max([Schedule.spaced("2 seconds"), Schedule.recurs(5)]),
    }),
  );

/** The profile's GCP environment, resolved the way {@link state} resolves it. */
export const stateEnvironment = () =>
  // Fresh per call, so an environment provided to this layer is not
  // shadowed by one memoized for another stack in the same run.
  Layer.fresh(fromCredentials()).pipe(
    Layer.provideMerge(Layer.fresh(GcpCredentials.fromAuthProvider())),
    Layer.provideMerge(GcpAuth),
    Layer.provideMerge(CredentialsStoreLive),
  );

/**
 * Create the state bucket with its secure defaults, or converge an existing
 * one to them: what first use of {@link state} does.
 */
export const provisionStateBucket = (
  bucket: string,
  project: string,
  location: string,
): Effect.Effect<void, StateStoreError, GcsDeps> =>
  ensureStateBucket(bucket, project, location).pipe(
    Effect.mapError(
      (cause) =>
        new StateStoreError({
          message: `Could not provision state bucket ${bucket}: ${cause._tag}`,
        }),
    ),
  );