sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ PubSub/ Topic.ts

import * as pubsub from "@distilled.cloud/gcp/pubsub_v1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { tagRecord } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { GcpEnvironment } from "../Environment.ts";
import {
  createInternalLabels,
  diffLabels,
  hasAlchemyLabels,
  stripInternalLabels,
  toLabels,
} from "../Labels.ts";
import type { Providers } from "../Providers.ts";

export type TopicProps = {
  /**
   * Topic id (the `{topic}` segment of `projects/{project}/topics/{topic}`).
   * If omitted, a unique name is generated from the stack, stage, and
   * logical id.
   */
  topicId?: string;
  /**
   * User labels. Alchemy ownership labels are merged in automatically.
   */
  labels?: Record<string, string>;
  /**
   * Cloud KMS key used to encrypt messages, as
   * `projects/{project}/locations/{location}/keyRings/{keyRing}/cryptoKeys/{cryptoKey}`.
   */
  kmsKeyName?: string;
  /**
   * Minimum duration to retain published messages (e.g. `"86400s"`).
   */
  messageRetentionDuration?: string;
};

export type Topic = Resource<
  "GCP.PubSub.Topic",
  TopicProps,
  {
    /** Full resource name `projects/{project}/topics/{topic}`. */
    name: string;
    /** Topic id (last path segment). */
    topicId: string;
    /** Project id. */
    project: string;
    /** User labels (Alchemy ownership labels stripped). */
    labels: Record<string, string>;
    /** KMS key used for encryption, if any. */
    kmsKeyName: string | undefined;
    /** Message retention duration, if set. */
    messageRetentionDuration: string | undefined;
    /** Server-reported topic state. */
    state: string | undefined;
  },
  never,
  Providers
>;

/**
 * A Google Cloud Pub/Sub topic.
 *
 * ### Creating a Topic
 * **Example:** Generated name
 * ```typescript
 * const topic = yield* GCP.PubSub.Topic("events", {});
 * ```
 *
 * **Example:** Explicit id and labels
 * ```typescript
 * const topic = yield* GCP.PubSub.Topic("events", {
 *   topicId: "order-events",
 *   labels: { env: "prod" },
 * });
 * ```
 *
 * ### Updating a Topic
 * **Example:** Change labels
 * ```typescript
 * const topic = yield* GCP.PubSub.Topic("events", {
 *   labels: { env: "prod" },
 * });
 * ```
 *
 * ### Binding from a Function
 * **Example:** Publish from Cloud Run
 * ```typescript
 * export class Api extends GCP.Function<Api>()(
 *   "Api",
 *   { main: import.meta.url },
 *   Effect.gen(function* () {
 *     const topic = yield* GCP.PubSub.Topic("events", {});
 *     const publish = yield* GCP.PubSub.Publish(topic);
 *     return {
 *       fetch: Effect.gen(function* () {
 *         yield* publish({
 *           body: { messages: [{ data: btoa("hello") }] },
 *         }).pipe(Effect.orDie);
 *         return HttpServerResponse.text("ok");
 *       }),
 *     };
 *   }).pipe(Effect.provide([GCP.PubSub.PublishHttp])),
 * ) {}
 * ```
 *
 * ### Destroying a Topic
 * **Example:** `alchemy destroy` deletes the topic after its subscriptions.
 *
 * @resource
 * @category PubSub
 */
export const Topic = Resource<Topic>("GCP.PubSub.Topic");

export class TopicStillExists extends Data.TaggedError("GCP.PubSub.TopicStillExists")<{
  resourceName: string;
}> {}

export class TopicNotResolved extends Data.TaggedError("GCP.PubSub.TopicNotResolved")<{
  resourceName: string;
}> {}

const topicIdOf = (name: string) => name.split("/").pop() ?? name;

const resourceName = (project: string, topicId: string) => `projects/${project}/topics/${topicId}`;

const userLabels = (
  labels: Record<string, string | undefined> | null | undefined,
): Record<string, string> => stripInternalLabels(tagRecord(labels));

const toId = (id: string, topicId: string | undefined, existing?: string) =>
  Effect.gen(function* () {
    return (
      topicId ?? existing ?? (yield* createPhysicalName({ id, maxLength: 255, lowercase: true }))
    );
  });

const toAttrs = (topic: pubsub.Topic, project: string) => {
  const name = topic.name ?? "";
  return {
    name,
    topicId: topicIdOf(name),
    project,
    labels: userLabels(topic.labels),
    kmsKeyName: topic.kmsKeyName,
    messageRetentionDuration: topic.messageRetentionDuration,
    state: topic.state,
  };
};

const getByName = (name: string) =>
  pubsub
    .getProjectsTopics({ topic: name })
    .pipe(Effect.catchTag("NotFound", () => Effect.undefined));

// Pub/Sub serves reads from replicas that lag writes: one successful GET
// does not mean the next one (or a publish routed to it) sees the
// resource. Require consecutive successful reads before reporting ready.
const CONSISTENT_READS = 3;

const waitUntilPresent = (name: string) =>
  Effect.gen(function* () {
    let found: pubsub.Topic | undefined;
    for (let read = 0; read < CONSISTENT_READS; read++) {
      if (read > 0) yield* Effect.sleep("1 second");
      found = yield* getByName(name);
      if (found === undefined) {
        return yield* new TopicNotResolved({ resourceName: name });
      }
    }
    return found!;
  }).pipe(
    Effect.retry({
      while: (error) => error._tag === "GCP.PubSub.TopicNotResolved",
      schedule: Schedule.spaced("1 second"),
      times: 60,
    }),
  );

export const TopicProvider = () =>
  Provider.succeed(Topic, {
    stables: ["name", "topicId", "project"],

    // topicId is the physical identity: a new id is a new topic, and the
    // old one must be deleted (not left behind by an in-place "update").
    diff: Effect.fn(function* ({ id, news, olds, output }) {
      if (!isResolved(news)) return undefined;
      const previous = output?.topicId ?? olds?.topicId;
      if (previous === undefined) return undefined;
      const next = yield* toId(id, news.topicId, output?.topicId);
      return next !== previous ? { action: "replace" as const, deleteFirst: false } : undefined;
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const topicId = yield* toId(id, olds?.topicId, output?.topicId);
      const name = output?.name ?? resourceName(env.project, topicId);
      const existing = yield* getByName(name);
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing, env.project);
      return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        return yield* pubsub.listProjectsTopics
          .pages({
            project: `projects/${env.project}`,
            pageSize: 1000,
          })
          .pipe(
            Stream.flatMap((page) => Stream.fromIterable(page.topics ?? [])),
            Stream.filter((topic) =>
              Object.keys(topic.labels ?? {}).some((key) => key.startsWith("alchemy-")),
            ),
            Stream.map((topic) => toAttrs(topic, env.project)),
            Stream.runCollect,
            Effect.map((chunk) => Array.from(chunk)),
          );
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const topicId = yield* toId(id, news.topicId, output?.topicId);
      const name = resourceName(env.project, topicId);
      const desiredLabels = {
        ...toLabels(news.labels),
        ...(yield* createInternalLabels(id)),
      };

      let current = yield* getByName(name);

      if (current === undefined) {
        yield* pubsub
          .createProjectsTopics({
            name,
            body: {
              labels: desiredLabels,
              kmsKeyName: news.kmsKeyName,
              messageRetentionDuration: news.messageRetentionDuration,
            },
          })
          .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
        // Block until Pub/Sub serves the topic, not just the create response.
        current = yield* waitUntilPresent(name);
      }

      if (current === undefined) {
        return yield* new TopicNotResolved({ resourceName: name });
      }

      const observedLabels = tagRecord(current.labels);
      const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
      const labelsChanged = upsert.length > 0 || removed.length > 0;
      const kmsChanged = (current.kmsKeyName ?? "") !== (news.kmsKeyName ?? "");
      const retentionChanged =
        (current.messageRetentionDuration ?? "") !== (news.messageRetentionDuration ?? "");

      if (labelsChanged || kmsChanged || retentionChanged) {
        current = yield* pubsub.patchProjectsTopics({
          name,
          body: {
            topic: {
              name,
              labels: desiredLabels,
              kmsKeyName: news.kmsKeyName,
              messageRetentionDuration: news.messageRetentionDuration,
            },
            updateMask: [
              labelsChanged ? "labels" : undefined,
              kmsChanged ? "kmsKeyName" : undefined,
              retentionChanged ? "messageRetentionDuration" : undefined,
            ]
              .filter((field): field is string => field !== undefined)
              .join(","),
          },
        });
      }

      return toAttrs(current, env.project);
    }),

    delete: Effect.fn(function* ({ output }) {
      yield* pubsub
        .deleteProjectsTopics({ topic: output.name })
        .pipe(Effect.catchTag("NotFound", () => Effect.void));
      // Pub/Sub reads lag deletes; block until it stops serving it.
      yield* getByName(output.name).pipe(
        Effect.filterOrFail(
          (existing) => existing === undefined,
          () => new TopicStillExists({ resourceName: output.name }),
        ),
        Effect.retry({
          while: (error) => error._tag === "GCP.PubSub.TopicStillExists",
          schedule: Schedule.spaced("2 seconds"),
          times: 30,
        }),
      );
    }),
  });