sailfork / t3code-github / .repos/ alchemy-effect/ examples/ gcp-event-pipeline/ src/ Drain.ts

import * as GCP from "@alchemy.run/gcp";
import * as Effect from "effect/Effect";
import * as Option from "effect/Option";
import { EventsTable, Inbox, type EventRow } from "./resources.ts";

/** One pull returns at most this many messages. */
const BATCH = 100;

/**
 * The batch half of the pipeline: drain Pub/Sub into BigQuery.
 *
 * A Cloud Run Job is the right host for this. It runs to completion and
 * exits, so nothing is billed between drains, and Cloud Scheduler or an
 * operator can start it whenever a batch is due.
 *
 * The order matters: insert first, ack second. A crash between them
 * redelivers the batch and duplicates rows, which a `SELECT DISTINCT id`
 * removes; acking first would lose events outright.
 */
export default class Drain extends GCP.Run.Job<Drain>()(
  "Drain",
  {
    main: import.meta.url,
  },
  Effect.gen(function* () {
    const inbox = yield* Inbox;
    const table = yield* EventsTable;

    // pubsub.subscriber on the subscription, and bigquery.dataEditor on
    // the table — nothing on the project.
    const subscription = yield* GCP.PubSub.ReadSubscription(inbox);
    const warehouse = yield* GCP.BigQuery.WriteTable(table);

    return {
      // A pull may return fewer messages than are waiting, so drain
      // batch by batch until one comes back empty.
      run: Effect.gen(function* () {
        // A pull waits for messages; an empty subscription answers
        // nothing, so a quiet 10 seconds means the backlog is drained.
        const received = yield* subscription
          .pull({ maxMessages: BATCH })
          .pipe(Effect.timeoutOption("10 seconds"));
        const messages = Option.getOrElse(received, () => []);
        if (messages.length === 0) {
          yield* Effect.log("drain: nothing left");
          return 0;
        }

        const rows = messages.map((message) => JSON.parse(message.text) as EventRow);

        // insertIds make the streaming insert idempotent inside
        // BigQuery's dedup window, so a redelivered batch collapses.
        yield* warehouse.insert(rows, { insertIds: rows.map((row) => row.id) });

        yield* subscription.acknowledge(messages.map((message) => message.ackId));

        yield* Effect.log(`drain: wrote ${rows.length} row(s)`);
        return messages.length;
      }).pipe(Effect.repeat({ until: (count) => count === 0 }), Effect.asVoid, Effect.orDie),
    };
  }).pipe(Effect.provide([GCP.PubSub.ReadSubscriptionHttp, GCP.BigQuery.WriteTableHttp])),
) {}