sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ Storage/ BucketEventSource.ts

import type * as storage from "@distilled.cloud/gcp/storage_v1";
import * as Binding from "alchemy/Binding";
import * as Effect from "effect/Effect";
import type * as Stream from "effect/Stream";
import type { TopicEventSourceProps, TopicMessage } from "../PubSub/TopicEventSource.ts";
import type { Bucket } from "./Bucket.ts";

/** Cloud Storage object event types delivered through Pub/Sub. */
export type BucketEventType =
  | "OBJECT_FINALIZE"
  | "OBJECT_DELETE"
  | "OBJECT_ARCHIVE"
  | "OBJECT_METADATA_UPDATE";

/** One Cloud Storage object change. */
export interface BucketEvent {
  /** What happened to the object. */
  eventType: BucketEventType;
  /** Bucket the object lives in. */
  bucket: string;
  /** Object name (key). */
  object: string;
  /** Object generation the event refers to. */
  generation: string;
  /** RFC3339 time the event occurred. */
  eventTime: string;
  /**
   * Object metadata at the time of the event (the `JSON_API_V1` payload):
   * `size`, `contentType`, `md5Hash`, `metadata`, …
   */
  metadata: storage.Storage_Object;
}

export interface BucketEventSourceProps extends Omit<TopicEventSourceProps, "filter"> {
  /**
   * Event types to deliver. Filtered by the bucket notification, so other
   * events never reach the host.
   * @default ["OBJECT_FINALIZE"]
   */
  eventTypes?: BucketEventType[];
  /**
   * Only deliver events for objects whose names start with this prefix.
   * Filtered by the bucket notification.
   */
  prefix?: string;
}

export type BucketEventsHandler<Req> = (
  events: Stream.Stream<BucketEvent>,
) => Effect.Effect<void, never, Req>;

export type BucketEventSourceService = <Req = never>(
  bucket: Bucket,
  props: BucketEventSourceProps,
  process: BucketEventsHandler<Req>,
) => Effect.Effect<void, never, never>;

/**
 * Event source streaming a Cloud Storage {@link Bucket}'s object changes
 * (created, deleted, archived, metadata updated) into the hosting compute.
 *
 * `GCP.Storage.BucketEventSourceLive` provisions a Pub/Sub topic owned by
 * the host plus a `JSON_API_V1` bucket notification on it (filtered by
 * event type and object prefix), then delegates delivery to whichever
 * `GCP.PubSub.TopicEventSource` implementation is provided alongside it:
 * `GCP.Run.TopicEventSource` (push, for `GCP.Function` /
 * `GCP.CloudFunctions.Function`) or `GCP.Run.TopicPullEventSource` (pull,
 * for `GCP.Run.Job` / `GCP.Run.WorkerPool`).
 *
 * Consume it through {@link consumeBucketEvents}.
 *
 * ### Consuming Bucket Events
 * **Example:** Process uploads on a Cloud Run service
 * ```typescript
 * export class Thumbnails extends GCP.Function<Thumbnails>()(
 *   "Thumbnails",
 *   { main: import.meta.url },
 *   Effect.gen(function* () {
 *     const uploads = yield* GCP.Storage.Bucket("Uploads", {});
 *     yield* GCP.Storage.consumeBucketEvents(
 *       uploads,
 *       { prefix: "incoming/" },
 *       (events) =>
 *         events.pipe(
 *           Stream.runForEach((event) =>
 *             Effect.log(`${event.object} (${event.metadata.size} bytes)`),
 *           ),
 *         ),
 *     );
 *   }).pipe(
 *     Effect.provide(GCP.Storage.BucketEventSourceLive),
 *     Effect.provide(GCP.Run.TopicEventSource),
 *   ),
 * ) {}
 * ```
 *
 * **Example:** Deletions on a worker pool
 * ```typescript
 * yield* GCP.Storage.consumeBucketEvents(
 *   uploads,
 *   { eventTypes: ["OBJECT_DELETE", "OBJECT_ARCHIVE"] },
 *   (events) => events.pipe(Stream.runForEach(handle)),
 * );
 * // …provided with GCP.Storage.BucketEventSourceLive
 * // and GCP.Run.TopicPullEventSource
 * ```
 *
 * @binding
 * @category Storage
 */
export interface BucketEventSource extends Binding.Service<
  BucketEventSource,
  "GCP.Storage.BucketEventSource",
  BucketEventSourceService
> {}

export const BucketEventSource = Binding.Service<BucketEventSource>(
  "GCP.Storage.BucketEventSource",
);

/**
 * Subscribe an Effect handler to object changes in a Cloud Storage
 * {@link Bucket}. See {@link BucketEventSource} for the implementations.
 */
export function consumeBucketEvents<Req = never, ResourceReq = never>(
  bucket: Bucket | Effect.Effect<Bucket, never, ResourceReq>,
  process: BucketEventsHandler<Req>,
): Effect.Effect<void, never, BucketEventSource | ResourceReq>;
export function consumeBucketEvents<Req = never, ResourceReq = never>(
  bucket: Bucket | Effect.Effect<Bucket, never, ResourceReq>,
  props: BucketEventSourceProps,
  process: BucketEventsHandler<Req>,
): Effect.Effect<void, never, BucketEventSource | ResourceReq>;
export function consumeBucketEvents<Req = never, ResourceReq = never>(
  bucket: Bucket | Effect.Effect<Bucket, never, ResourceReq>,
  propsOrProcess: BucketEventSourceProps | BucketEventsHandler<Req>,
  maybeProcess?: BucketEventsHandler<Req>,
): Effect.Effect<void, never, BucketEventSource | ResourceReq> {
  const [props, process] =
    typeof propsOrProcess === "function"
      ? [{} as BucketEventSourceProps, propsOrProcess]
      : [propsOrProcess, maybeProcess!];
  // Accept the resource or the Effect that declares it, like bindings do.
  const resolved = Effect.isEffect(bucket) ? bucket : Effect.succeed(bucket);
  return resolved.pipe(
    Effect.flatMap((value) => BucketEventSource.use((source) => source(value, props, process))),
  );
}

const decodeBase64Utf8 = (data: string) => {
  const binary = atob(data);
  const bytes = new Uint8Array(binary.length);
  for (let index = 0; index < binary.length; index++) {
    bytes[index] = binary.charCodeAt(index);
  }
  return new TextDecoder().decode(bytes);
};

/**
 * Parse a Cloud Storage Pub/Sub notification into a {@link BucketEvent}.
 * The routing fields come from the message attributes; `metadata` is the
 * `JSON_API_V1` object resource in `data` (empty when the notification
 * uses `NONE` or the payload is unreadable).
 */
export const toBucketEvent = ({ message }: TopicMessage) =>
  Effect.sync((): BucketEvent => {
    const attributes = message.attributes ?? {};
    let metadata: storage.Storage_Object = {};
    try {
      if (message.data) {
        metadata = JSON.parse(decodeBase64Utf8(message.data)) as storage.Storage_Object;
      }
    } catch {
      metadata = {};
    }
    return {
      eventType: (attributes.eventType ?? "OBJECT_FINALIZE") as BucketEventType,
      bucket: attributes.bucketId ?? metadata.bucket ?? "",
      object: attributes.objectId ?? metadata.name ?? "",
      generation: attributes.objectGeneration ?? String(metadata.generation ?? ""),
      eventTime: attributes.eventTime ?? message.publishTime ?? "",
      metadata,
    };
  });