sailfork / t3code-github / .repos/ alchemy-effect/ examples/ gcp-storage-uploads/ src/ Indexer.ts

import * as GCP from "@alchemy.run/gcp";
import * as Effect from "effect/Effect";
import * as HttpServerResponse from "effect/http/HttpServerResponse";
import * as Stream from "effect/Stream";
import { documentFor, Files, UPLOADS_PREFIX, Uploads, type FileRecord } from "./resources.ts";

const sha256Hex = (bytes: Uint8Array) =>
  Effect.promise(() => crypto.subtle.digest("SHA-256", bytes as Uint8Array<ArrayBuffer>)).pipe(
    Effect.map((digest) =>
      Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join(""),
    ),
  );

/**
 * The background half of the pipeline: index every finished upload.
 *
 * Cloud Storage publishes an `OBJECT_FINALIZE` notification for each
 * object written under `uploads/`; Pub/Sub pushes it to this service,
 * which reads the object back, hashes it, and records the result in
 * Firestore. Nothing here runs on the upload's request path, so a slow
 * hash of a large file never delays the client.
 *
 * The service stays private: Pub/Sub authenticates each push with an
 * OIDC token for the service's own identity.
 */
export default class Indexer extends GCP.Function<Indexer>()(
  "Indexer",
  {
    main: import.meta.url,
  },
  Effect.gen(function* () {
    const bucket = yield* Uploads;
    const database = yield* Files;

    const uploads = yield* GCP.Storage.ReadBucket(bucket);
    const files = yield* GCP.Firestore.WriteDatabase(database);

    yield* GCP.Storage.consumeBucketEvents(
      bucket,
      { eventTypes: ["OBJECT_FINALIZE"], prefix: UPLOADS_PREFIX },
      (events) =>
        events.pipe(
          Stream.runForEach((event) =>
            Effect.gen(function* () {
              // Read the exact generation the event describes. If it has
              // since been overwritten or deleted, a later event (or the
              // delete) owns the record, so there is nothing to do.
              const object = yield* uploads.get(event.object, {
                generation: event.generation,
              });
              if (object === undefined) {
                yield* Effect.log(`indexer: ${event.object} is gone, skipping`);
                return;
              }

              const name = event.object.slice(UPLOADS_PREFIX.length);
              const record: FileRecord = {
                name,
                object: event.object,
                generation: event.generation,
                size: object.body.byteLength,
                contentType:
                  object.contentType ?? event.metadata.contentType ?? "application/octet-stream",
                sha256: yield* sha256Hex(object.body),
                indexedAt: new Date(),
              };

              yield* files.set(documentFor(name), { ...record });
              yield* Effect.log(`indexer: ${name} (${record.size} bytes)`);
            }).pipe(
              // A failure fails the push, so Pub/Sub redelivers the event.
              Effect.orDie,
            ),
          ),
        ),
    );

    return {
      fetch: Effect.succeed(HttpServerResponse.text("ok")),
    };
  }).pipe(
    Effect.provide([
      GCP.Storage.BucketEventSourceLive,
      GCP.Storage.ReadBucketHttp,
      GCP.Firestore.WriteDatabaseHttp,
    ]),
    Effect.provide(GCP.Run.TopicEventSource),
  ),
) {}