sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ PubSub/ Snapshot.ts

import * as pubsub from "@distilled.cloud/gcp/pubsub_v1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { tagRecord } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { GcpEnvironment } from "../Environment.ts";
import {
  createInternalLabels,
  diffLabels,
  hasAlchemyLabels,
  stripInternalLabels,
  toLabels,
} from "../Labels.ts";
import type { Providers } from "../Providers.ts";

export type SnapshotProps = {
  /**
   * Snapshot id (the `{snapshot}` segment of
   * `projects/{project}/snapshots/{snapshot}`). If omitted, a unique name
   * is generated from the stack, stage, and logical id. Changing it
   * replaces the snapshot.
   */
  snapshotId?: string;
  /**
   * Subscription whose backlog this snapshot retains. Accepts a full
   * resource name (`projects/{project}/subscriptions/{subscription}`) or
   * a subscription id. Immutable — changing it replaces the snapshot.
   */
  subscription: string;
  /**
   * User labels. Alchemy ownership labels are merged in automatically.
   */
  labels?: Record<string, string>;
};

export type Snapshot = Resource<
  "GCP.PubSub.Snapshot",
  SnapshotProps,
  {
    /** Full resource name `projects/{project}/snapshots/{snapshot}`. */
    name: string;
    /** Snapshot id (last path segment). */
    snapshotId: string;
    /** Project id. */
    project: string;
    /** Topic this snapshot is retaining messages from. */
    topic: string | undefined;
    /** RFC3339 time after which the snapshot is no longer guaranteed. */
    expireTime: string | undefined;
    /** User labels (Alchemy ownership labels stripped). */
    labels: Record<string, string>;
  },
  never,
  Providers
>;

/**
 * A Google Cloud Pub/Sub snapshot of a subscription's message backlog.
 *
 * Snapshots capture unacked messages at creation time (and subsequent
 * publishes to the topic) so a subscription can later `Seek` back to that
 * point. The source `subscription` is immutable — changing it replaces the
 * snapshot. Labels can be updated in place.
 *
 * ### Creating a Snapshot
 * **Example:** Snapshot of a pull subscription
 * ```typescript
 * const topic = yield* GCP.PubSub.Topic("events", {});
 * const subscription = yield* GCP.PubSub.Subscription("orders", {
 *   topic: topic.name,
 * });
 * const snapshot = yield* GCP.PubSub.Snapshot("checkpoint", {
 *   subscription: subscription.name,
 * });
 * ```
 *
 * **Example:** Explicit id and labels
 * ```typescript
 * const topic = yield* GCP.PubSub.Topic("events", {});
 * const subscription = yield* GCP.PubSub.Subscription("orders", {
 *   topic: topic.name,
 * });
 * const snapshot = yield* GCP.PubSub.Snapshot("checkpoint", {
 *   snapshotId: "order-checkpoint",
 *   subscription: subscription.name,
 *   labels: { env: "prod" },
 * });
 * ```
 *
 * @resource
 * @category PubSub
 */
export const Snapshot = Resource<Snapshot>("GCP.PubSub.Snapshot");

export class SnapshotNotResolved extends Data.TaggedError("GCP.PubSub.SnapshotNotResolved")<{
  resourceName: string;
}> {}

export class SnapshotStillExists extends Data.TaggedError("GCP.PubSub.SnapshotStillExists")<{
  resourceName: string;
}> {}

const snapshotIdOf = (name: string) => name.split("/").pop() ?? name;

const resourceName = (project: string, snapshotId: string) =>
  `projects/${project}/snapshots/${snapshotId}`;

const subscriptionNameOf = (project: string, subscription: string) =>
  subscription.startsWith("projects/")
    ? subscription
    : `projects/${project}/subscriptions/${subscription}`;

const resourceKey = (value: string | undefined) =>
  value === undefined ? undefined : snapshotIdOf(value);

const userLabels = (
  labels: Record<string, string | undefined> | null | undefined,
): Record<string, string> => stripInternalLabels(tagRecord(labels));

const toId = (id: string, snapshotId: string | undefined, existing?: string) =>
  Effect.gen(function* () {
    if (snapshotId !== undefined) return snapshotId;
    if (existing !== undefined) return existing;
    const generated = yield* createPhysicalName({
      id,
      maxLength: 255,
      lowercase: true,
      forbiddenPrefixes: ["goog"],
    });
    return /^[a-z]/.test(generated) ? generated : `s${generated}`.slice(0, 255);
  });

const toAttrs = (snapshot: pubsub.Snapshot, project: string) => {
  const name = snapshot.name ?? "";
  return {
    name,
    snapshotId: snapshotIdOf(name),
    project,
    topic: snapshot.topic,
    expireTime: snapshot.expireTime,
    labels: userLabels(snapshot.labels),
  };
};

const getByName = (name: string) =>
  pubsub
    .getProjectsSnapshots({ snapshot: name })
    .pipe(Effect.catchTag("NotFound", () => Effect.undefined));

const waitUntilGone = (name: string) =>
  getByName(name).pipe(
    Effect.filterOrFail(
      (existing): existing is undefined => existing === undefined,
      () => new SnapshotStillExists({ resourceName: name }),
    ),
    Effect.asVoid,
    Effect.retry({
      while: (error) => error._tag === "GCP.PubSub.SnapshotStillExists",
      // Pub/Sub reads lag deletes; allow up to a minute.
      schedule: Schedule.spaced("2 seconds"),
      times: 30,
    }),
  );

const CONSISTENT_READS = 3;

const waitUntilPresent = (name: string) =>
  Effect.gen(function* () {
    let found: pubsub.Snapshot | undefined;
    for (let read = 0; read < CONSISTENT_READS; read++) {
      if (read > 0) yield* Effect.sleep("1 second");
      found = yield* getByName(name);
      if (found === undefined) return yield* new SnapshotNotResolved({ resourceName: name });
    }
    return found!;
  }).pipe(
    Effect.retry({
      while: (error) => error._tag === "GCP.PubSub.SnapshotNotResolved",
      schedule: Schedule.spaced("1 second"),
      times: 60,
    }),
  );

export const SnapshotProvider = () =>
  Provider.succeed(Snapshot, {
    stables: ["name", "snapshotId", "project", "topic"],

    diff: Effect.fn(function* ({ news, olds, output }) {
      if (!isResolved(news)) return undefined;
      const previousId = olds?.snapshotId ?? output?.snapshotId;
      const nameChanged =
        news.snapshotId !== undefined && previousId !== undefined && news.snapshotId !== previousId;
      const previousSubscription = olds?.subscription;
      const subscriptionChanged =
        previousSubscription !== undefined &&
        resourceKey(news.subscription) !== resourceKey(previousSubscription);
      if (!nameChanged && !subscriptionChanged) {
        return undefined;
      }
      return {
        action: "replace" as const,
        deleteFirst:
          !nameChanged &&
          previousId !== undefined &&
          (news.snapshotId ?? previousId) === previousId,
      };
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const snapshotId = yield* toId(id, olds?.snapshotId, output?.snapshotId);
      const name = output?.name ?? resourceName(env.project, snapshotId);
      const existing = yield* getByName(name);
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing, env.project);
      return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        return yield* pubsub.listProjectsSnapshots
          .pages({
            project: `projects/${env.project}`,
            pageSize: 1000,
          })
          .pipe(
            Stream.flatMap((page) => Stream.fromIterable(page.snapshots ?? [])),
            Stream.filter((snapshot) =>
              Object.keys(snapshot.labels ?? {}).some((key) => key.startsWith("alchemy-")),
            ),
            Stream.map((snapshot) => toAttrs(snapshot, env.project)),
            Stream.runCollect,
            Effect.map((chunk) => Array.from(chunk)),
          );
      }),

    reconcile: Effect.fn(function* ({ id, news, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const snapshotId = yield* toId(id, news.snapshotId, output?.snapshotId);
      const name = resourceName(env.project, snapshotId);
      const subscriptionName = subscriptionNameOf(env.project, news.subscription);
      const desiredLabels = {
        ...toLabels(news.labels),
        ...(yield* createInternalLabels(id)),
      };
      // Snapshot GET does not return the source subscription. Recreate when
      // persisted props show it changed — the field is immutable.
      const subscriptionChanged =
        olds?.subscription !== undefined &&
        resourceKey(news.subscription) !== resourceKey(olds.subscription);

      let current = yield* getByName(name);
      if (current !== undefined && subscriptionChanged) {
        yield* pubsub
          .deleteProjectsSnapshots({ snapshot: name })
          .pipe(Effect.catchTag("NotFound", () => Effect.void));
        yield* waitUntilGone(name);
        current = undefined;
      }

      if (current === undefined) {
        yield* pubsub
          .createProjectsSnapshots({
            name,
            body: {
              subscription: subscriptionName,
              labels: desiredLabels,
            },
          })
          .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
        // Block until Pub/Sub consistently serves the snapshot; the next
        // call (label patch) can otherwise hit a lagging replica.
        current = yield* waitUntilPresent(name);
      }

      if (current === undefined) {
        return yield* new SnapshotNotResolved({ resourceName: name });
      }

      const observedLabels = tagRecord(current.labels);
      const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
      if (upsert.length > 0 || removed.length > 0) {
        current = yield* pubsub.patchProjectsSnapshots({
          name,
          body: {
            snapshot: {
              name,
              labels: desiredLabels,
            },
            updateMask: "labels",
          },
        });
      }

      return toAttrs(current, env.project);
    }),

    delete: Effect.fn(function* ({ output }) {
      yield* pubsub
        .deleteProjectsSnapshots({ snapshot: output.name })
        .pipe(Effect.catchTag("NotFound", () => Effect.void));
      yield* waitUntilGone(output.name);
    }),
  });