sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ Storage/ Bucket.ts

import * as storage from "@distilled.cloud/gcp/storage_v1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { tagRecord } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { GcpEnvironment } from "../Environment.ts";
import {
  createInternalLabels,
  diffLabels,
  hasAlchemyLabels,
  stripInternalLabels,
  toLabels,
} from "../Labels.ts";
import type { Providers } from "../Providers.ts";

export type BucketProps = {
  /**
   * Globally-unique bucket name. If omitted, a unique name is generated from
   * the stack, stage, and logical id. Must be 3-63 characters, lowercase,
   * and DNS compatible.
   */
  bucketName?: string;
  /**
   * Location of the bucket (`US`, `EU`, `US-CENTRAL1`, …). Immutable —
   * changing it replaces the bucket.
   * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
   */
  location?: string;
  /**
   * Default storage class for new objects.
   * @default "STANDARD"
   */
  storageClass?: string;
  /**
   * Whether object versioning is enabled.
   * @default false
   */
  versioning?: boolean;
  /**
   * User labels. Alchemy ownership labels are merged in automatically.
   */
  labels?: Record<string, string>;
  /**
   * Delete all objects (and versions) before destroying the bucket.
   * @default false
   */
  forceDestroy?: boolean;
  /**
   * Enable uniform bucket-level access. Required for managed folders.
   * Hierarchical namespace implies this is enabled. Fine-grained ACLs
   * cannot be used when this is true.
   * @default false
   */
  uniformBucketLevelAccess?: boolean;
  /**
   * Enable hierarchical namespace (first-class folders). Immutable —
   * must be set at create time; changing it replaces the bucket.
   * Implies uniform bucket-level access.
   * @default false
   */
  hierarchicalNamespace?: boolean;
  /**
   * Static website configuration. Cloud Storage applies it when the bucket
   * is served through a CNAME or an external HTTPS load balancer (backend
   * bucket) — not on `storage.googleapis.com/{bucket}/…` path-style URLs.
   * Omitting it removes the configuration.
   */
  website?: BucketWebsite;
};

export interface BucketWebsite {
  /**
   * Object suffix appended to directory-style requests, e.g. `index.html`
   * so `/docs/` serves `docs/index.html`.
   */
  mainPageSuffix?: string;
  /** Object served (with status 404) when a requested object is missing. */
  notFoundPage?: string;
}

export type Bucket = Resource<
  "GCP.Storage.Bucket",
  BucketProps,
  {
    /** Globally unique bucket name. */
    bucketName: string;
    /** Location of the bucket. */
    location: string;
    /** Location type (`region`, `multi-region`, `dual-region`). */
    locationType: string | undefined;
    /** Default storage class. */
    storageClass: string | undefined;
    /** Whether object versioning is enabled. */
    versioning: boolean;
    /** User labels (Alchemy ownership labels stripped). */
    labels: Record<string, string>;
    /** GCS self-link. */
    selfLink: string | undefined;
    /** Numeric project that owns the bucket. */
    projectNumber: string | undefined;
    /** RFC3339 creation timestamp. */
    timeCreated: string | undefined;
    /** Whether uniform bucket-level access is enabled. */
    uniformBucketLevelAccess: boolean;
    /** Whether hierarchical namespace is enabled. */
    hierarchicalNamespace: boolean;
    /** Static website configuration, if set. */
    website: BucketWebsite | undefined;
  },
  never,
  Providers
>;

/**
 * A Google Cloud Storage bucket.
 *
 * ### Creating a Bucket
 * **Example:** Generated name
 * ```typescript
 * const bucket = yield* GCP.Storage.Bucket("assets", {
 *   forceDestroy: true,
 * });
 * ```
 *
 * **Example:** Explicit name, location, and labels
 * ```typescript
 * const bucket = yield* GCP.Storage.Bucket("assets", {
 *   bucketName: "my-app-assets",
 *   location: "US-CENTRAL1",
 *   storageClass: "STANDARD",
 *   versioning: true,
 *   labels: { env: "prod" },
 *   forceDestroy: true,
 * });
 * ```
 *
 * **Example:** Hierarchical namespace (folders)
 * ```typescript
 * const bucket = yield* GCP.Storage.Bucket("tree", {
 *   hierarchicalNamespace: true,
 *   uniformBucketLevelAccess: true,
 *   forceDestroy: true,
 * });
 * ```
 *
 * **Example:** Static website bucket with public read
 * ```typescript
 * const site = yield* GCP.Storage.Bucket("site", {
 *   uniformBucketLevelAccess: true,
 *   website: { mainPageSuffix: "index.html", notFoundPage: "404.html" },
 *   forceDestroy: true,
 * });
 * yield* GCP.IAM.Member("PublicRead", {
 *   kind: "storage.bucket",
 *   name: site.bucketName,
 *   role: "roles/storage.objectViewer",
 *   member: "allUsers",
 * });
 * ```
 *
 * ### Updating a Bucket
 * **Example:** Change labels
 * ```typescript
 * const bucket = yield* GCP.Storage.Bucket("assets", {
 *   forceDestroy: true,
 *   labels: { env: "prod" },
 * });
 * ```
 *
 * ### Binding from a Function
 * **Example:** PutObject from Cloud Run
 * ```typescript
 * export class Api extends GCP.Function<Api>()(
 *   "Api",
 *   { main: import.meta.url },
 *   Effect.gen(function* () {
 *     const bucket = yield* GCP.Storage.Bucket("assets", { forceDestroy: true });
 *     const putObject = yield* GCP.Storage.PutObject(bucket);
 *     return {
 *       fetch: Effect.gen(function* () {
 *         yield* putObject({ name: "hello.txt", body: "Hello!" }).pipe(
 *           Effect.orDie,
 *         );
 *         return HttpServerResponse.text("ok");
 *       }),
 *     };
 *   }).pipe(Effect.provide([GCP.Storage.PutObjectHttp])),
 * ) {}
 * ```
 *
 * ### Destroying a Bucket
 * **Example:** Empty then delete
 * ```typescript
 * const bucket = yield* GCP.Storage.Bucket("assets", {
 *   forceDestroy: true,
 * });
 * ```
 * `forceDestroy: true` deletes objects before the bucket. `alchemy destroy` waits until the bucket is gone.
 *
 * @resource
 * @category Storage
 */
export const Bucket = Resource<Bucket>("GCP.Storage.Bucket");

export class BucketNotResolved extends Data.TaggedError("GCP.Storage.BucketNotResolved")<{
  bucketName: string;
}> {}

const userLabels = (
  labels: Record<string, string | undefined> | null | undefined,
): Record<string, string> => stripInternalLabels(tagRecord(labels));

const toName = (id: string, name: string | undefined, existing?: string) =>
  Effect.gen(function* () {
    return (
      name ??
      existing ??
      (yield* createPhysicalName({
        id,
        maxLength: 63,
        lowercase: true,
      }))
    );
  });

const toAttrs = (bucket: storage.Bucket) => ({
  bucketName: bucket.name ?? bucket.id ?? "",
  location: bucket.location ?? "",
  locationType: bucket.locationType,
  storageClass: bucket.storageClass,
  versioning: bucket.versioning?.enabled === true,
  labels: userLabels(bucket.labels),
  selfLink: bucket.selfLink,
  projectNumber: bucket.projectNumber,
  timeCreated: bucket.timeCreated,
  uniformBucketLevelAccess: bucket.iamConfiguration?.uniformBucketLevelAccess?.enabled === true,
  hierarchicalNamespace: bucket.hierarchicalNamespace?.enabled === true,
  website: toWebsite(bucket.website),
});

const toWebsite = (website: storage.BucketWebsite | undefined): BucketWebsite | undefined =>
  website === undefined ||
  (website.mainPageSuffix === undefined && website.notFoundPage === undefined)
    ? undefined
    : {
        mainPageSuffix: website.mainPageSuffix,
        notFoundPage: website.notFoundPage,
      };

const getByName = (bucketName: string) =>
  storage
    .getBuckets({ bucket: bucketName, projection: "full" })
    .pipe(Effect.catchTag("NotFound", () => Effect.undefined));

const emptyBucket = (bucketName: string) =>
  storage.listObjects.items({ bucket: bucketName, versions: true, maxResults: 1000 }).pipe(
    Stream.runCollect,
    Effect.flatMap((chunk) =>
      Effect.forEach(
        chunk,
        (object) =>
          object.name
            ? storage
                .deleteObjects({
                  bucket: bucketName,
                  object: object.name,
                  generation: object.generation,
                })
                .pipe(Effect.catchTag("NotFound", () => Effect.void))
            : Effect.void,
        { concurrency: 8 },
      ),
    ),
    Effect.catchTag("NotFound", () => Effect.void),
  );

const emptyFolders = (bucketName: string) =>
  storage.listFolders.items({ bucket: bucketName, pageSize: 1000 }).pipe(
    Stream.runCollect,
    Effect.flatMap((chunk) => {
      const folders = Array.from(chunk)
        .map((folder) => folder.name)
        .filter((name): name is string => !!name)
        .sort((left, right) => right.length - left.length);
      return Effect.forEach(
        folders,
        (folder) =>
          storage
            .deleteFolders({ bucket: bucketName, folder })
            .pipe(Effect.catchTag("NotFound", () => Effect.void)),
        { concurrency: 1 },
      );
    }),
    Effect.catchTag("NotFound", () => Effect.void),
  );

const emptyManagedFolders = (bucketName: string) =>
  storage.listManagedFolders.items({ bucket: bucketName, pageSize: 1000 }).pipe(
    Stream.runCollect,
    Effect.flatMap((chunk) => {
      const folders = Array.from(chunk)
        .map((folder) => folder.name)
        .filter((name): name is string => !!name)
        .sort((left, right) => right.length - left.length);
      return Effect.forEach(
        folders,
        (managedFolder) =>
          storage
            .deleteManagedFolders({
              bucket: bucketName,
              managedFolder,
              allowNonEmpty: true,
            })
            .pipe(Effect.catchTag("NotFound", () => Effect.void)),
        { concurrency: 1 },
      );
    }),
    Effect.catchTag("NotFound", () => Effect.void),
  );

export const BucketProvider = () =>
  Provider.succeed(Bucket, {
    stables: ["bucketName", "location", "locationType", "projectNumber", "timeCreated", "selfLink"],

    diff: Effect.fn(function* ({ news, olds, output }) {
      if (!isResolved(news)) return undefined;
      // Bucket names are immutable: a new name is a new bucket, and the
      // old one must be deleted rather than left behind.
      const recordedName = output?.bucketName ?? olds?.bucketName;
      if (
        news.bucketName !== undefined &&
        recordedName !== undefined &&
        news.bucketName !== recordedName
      ) {
        return { action: "replace" as const, deleteFirst: false };
      }
      const previous = olds?.location ?? output?.location;
      const next = news.location;
      if (
        previous !== undefined &&
        next !== undefined &&
        previous.toUpperCase() !== next.toUpperCase()
      ) {
        const previousName = olds?.bucketName ?? output?.bucketName;
        const nextName = news.bucketName ?? previousName;
        return {
          action: "replace" as const,
          deleteFirst: nextName !== undefined && nextName === previousName,
        };
      }
      const previousHns = olds?.hierarchicalNamespace ?? output?.hierarchicalNamespace;
      const nextHns = news.hierarchicalNamespace === true;
      if (previousHns !== undefined && previousHns !== nextHns) {
        const previousName = olds?.bucketName ?? output?.bucketName;
        const nextName = news.bucketName ?? previousName;
        return {
          action: "replace" as const,
          deleteFirst: nextName !== undefined && nextName === previousName,
        };
      }
      return undefined;
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const bucketName = yield* toName(id, olds?.bucketName, output?.bucketName);
      const existing = yield* getByName(bucketName);
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing);
      return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        return yield* storage.listBuckets.items({ project: env.project, projection: "full" }).pipe(
          Stream.filter((bucket) =>
            Object.keys(bucket.labels ?? {}).some((key) => key.startsWith("alchemy-")),
          ),
          Stream.map(toAttrs),
          Stream.runCollect,
          Effect.map((chunk) => Array.from(chunk)),
        );
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const bucketName = yield* toName(id, news.bucketName, output?.bucketName);
      const location = news.location ?? output?.location ?? env.region.toUpperCase();
      const storageClass = news.storageClass ?? "STANDARD";
      const versioning = news.versioning === true;
      const hierarchicalNamespace = news.hierarchicalNamespace === true;
      const uniformBucketLevelAccess =
        news.uniformBucketLevelAccess === true || hierarchicalNamespace;
      const configureIam = news.uniformBucketLevelAccess !== undefined || hierarchicalNamespace;
      const desiredLabels = {
        ...toLabels(news.labels),
        ...(yield* createInternalLabels(id)),
      };
      const desiredWebsite = toWebsite(news.website);

      let current = yield* getByName(bucketName);

      if (current === undefined) {
        const created = yield* storage
          .insertBuckets({
            project: env.project,
            projection: "full",
            body: {
              name: bucketName,
              location,
              storageClass,
              versioning: { enabled: versioning },
              labels: desiredLabels,
              hierarchicalNamespace: hierarchicalNamespace ? { enabled: true } : undefined,
              website: desiredWebsite,
              iamConfiguration: configureIam
                ? {
                    uniformBucketLevelAccess: {
                      enabled: uniformBucketLevelAccess,
                    },
                  }
                : undefined,
            },
          })
          .pipe(Effect.catchTag("Conflict", () => getByName(bucketName)));
        current = created ?? undefined;
      }

      if (current === undefined) {
        return yield* new BucketNotResolved({ bucketName });
      }

      const observedLabels = tagRecord(current.labels);
      const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
      const labelsChanged = upsert.length > 0 || removed.length > 0;
      const storageClassChanged = (current.storageClass ?? "STANDARD") !== storageClass;
      const versioningChanged = (current.versioning?.enabled === true) !== versioning;
      const ublChanged =
        configureIam &&
        (current.iamConfiguration?.uniformBucketLevelAccess?.enabled === true) !==
          uniformBucketLevelAccess;
      const observedWebsite = toWebsite(current.website);
      const websiteChanged =
        observedWebsite?.mainPageSuffix !== desiredWebsite?.mainPageSuffix ||
        observedWebsite?.notFoundPage !== desiredWebsite?.notFoundPage;

      if (
        labelsChanged ||
        storageClassChanged ||
        versioningChanged ||
        ublChanged ||
        websiteChanged
      ) {
        const nextLabels: Record<string, string | null> = { ...desiredLabels };
        for (const key of removed) {
          nextLabels[key] = null;
        }
        current = yield* storage.patchBuckets({
          bucket: bucketName,
          projection: "full",
          body: {
            storageClass,
            versioning: { enabled: versioning },
            labels: nextLabels as unknown as Record<string, string>,
            iamConfiguration: ublChanged
              ? {
                  uniformBucketLevelAccess: {
                    enabled: uniformBucketLevelAccess,
                  },
                }
              : undefined,
            // PATCH merges nested objects and clears a field only when it
            // is sent as JSON null.
            website: websiteChanged
              ? ((desiredWebsite === undefined
                  ? null
                  : {
                      mainPageSuffix: desiredWebsite.mainPageSuffix ?? null,
                      notFoundPage: desiredWebsite.notFoundPage ?? null,
                    }) as unknown as storage.BucketWebsite)
              : undefined,
          },
        });
      }

      return toAttrs(current);
    }),

    delete: Effect.fn(function* ({ olds, output, force }) {
      const mayEmpty = olds.forceDestroy === true || force === true;
      const empty = Effect.gen(function* () {
        yield* emptyBucket(output.bucketName);
        if (output.hierarchicalNamespace) {
          yield* emptyFolders(output.bucketName);
        }
        if (output.uniformBucketLevelAccess) {
          yield* emptyManagedFolders(output.bucketName);
        }
      });
      // A writer that is still running (e.g. a Dataflow job staging temp
      // files) can add objects between the empty and the delete, which fails
      // with 409 "not empty"; with forceDestroy, empty again and retry.
      yield* Effect.gen(function* () {
        if (mayEmpty) yield* empty;
        yield* storage
          .deleteBuckets({ bucket: output.bucketName })
          .pipe(Effect.catchTag("NotFound", () => Effect.void));
      }).pipe(
        Effect.retry({
          while: (error) => mayEmpty && error._tag === "Conflict",
          times: 6,
          schedule: Schedule.spaced("10 seconds"),
        }),
      );
    }),
  });