sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ ManagedKafka/ ClustersTopic.ts

import * as kafka from "@distilled.cloud/gcp/managedkafka_v1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import { GcpEnvironment } from "../Environment.ts";
import type { Providers } from "../Providers.ts";
import {
  collectPages,
  expandParent,
  fieldMask,
  fingerprint,
  getTopic,
  listAlchemyClusters,
  normalizeLocation,
  parseName,
  replaceOnIdentity,
  retryTransient,
  rfc1035,
  stringMapOf,
  toPhysicalId,
} from "./internal.ts";

const DEFAULT_PARTITION_COUNT = 1;
const DEFAULT_REPLICATION_FACTOR = 3;

export type ClustersTopicProps = {
  /**
   * Parent cluster. Full name
   * `projects/{project}/locations/{location}/clusters/{cluster}` or the
   * cluster id (combined with `location`). Immutable — changing it
   * replaces the topic.
   */
  cluster: string;
  /**
   * Region used when `cluster` is a bare id.
   * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
   */
  location?: string;
  /**
   * Topic id (the `{topic}` segment of `.../clusters/{cluster}/topics/{topic}`).
   * If omitted, a unique RFC1035 name is generated. Immutable — changing
   * it replaces the topic.
   */
  topicId?: string;
  /**
   * Partition count. Can only be increased, not decreased.
   * @default 1
   */
  partitionCount?: number;
  /**
   * Replication factor. Immutable — changing it replaces the topic.
   * @default 3
   */
  replicationFactor?: number;
  /**
   * Topic configs that override cluster defaults (`cleanup.policy`,
   * `compression.type`, …).
   */
  configs?: Record<string, string>;
};

export type ClustersTopic = Resource<
  "GCP.ManagedKafka.ClustersTopic",
  ClustersTopicProps,
  {
    /** Full resource name `.../clusters/{cluster}/topics/{topic}`. */
    name: string;
    /** Topic id. */
    topicId: string;
    /** Parent cluster resource name. */
    cluster: string;
    /** Project id. */
    project: string;
    /** Location id. */
    location: string;
    /** Partition count. */
    partitionCount: number;
    /** Replication factor. */
    replicationFactor: number;
    /** Topic config overrides. */
    configs: Record<string, string>;
  },
  never,
  Providers
>;

/**
 * A Kafka topic on a Managed Service for Apache Kafka cluster.
 *
 * Changing `topicId`, `cluster`, `location`, or `replicationFactor`
 * replaces the topic. `partitionCount` can only increase. Configs update
 * in place.
 *
 * ### Creating a Topic
 * **Example:** Generated name
 * ```typescript
 * const topic = yield* GCP.ManagedKafka.ClustersTopic("Events", {
 *   cluster: cluster.name,
 * });
 * ```
 *
 * **Example:** Explicit id, partitions, and configs
 * ```typescript
 * const topic = yield* GCP.ManagedKafka.ClustersTopic("Events", {
 *   cluster: cluster.name,
 *   topicId: "order-events",
 *   partitionCount: 3,
 *   replicationFactor: 3,
 *   configs: { "cleanup.policy": "compact" },
 * });
 * ```
 *
 * @resource
 * @category ManagedKafka
 */
export const ClustersTopic = Resource<ClustersTopic>("GCP.ManagedKafka.ClustersTopic");

export class ClustersTopicNotResolved extends Data.TaggedError(
  "GCP.ManagedKafka.ClustersTopicNotResolved",
)<{
  resourceName: string;
}> {}

const clusterOf = (cluster: string, project: string, location: string) =>
  expandParent(cluster, project, location, "clusters");

const resourceName = (cluster: string, topicId: string) => `${cluster}/topics/${topicId}`;

const toAttrs = (topic: kafka.Topic, project: string) => {
  const name = topic.name ?? "";
  const parsed = parseName(name, "topics");
  return {
    name,
    topicId: parsed.id,
    cluster: parsed.parent,
    project: parsed.project || project,
    location: parsed.location,
    partitionCount: topic.partitionCount ?? DEFAULT_PARTITION_COUNT,
    replicationFactor: topic.replicationFactor ?? DEFAULT_REPLICATION_FACTOR,
    configs: stringMapOf(topic.configs),
  };
};

export const ClustersTopicProvider = () =>
  Provider.succeed(ClustersTopic, {
    stables: ["name", "topicId", "cluster", "project", "location"],

    diff: Effect.fn(function* ({ news, olds, output }) {
      if (!isResolved(news)) return undefined;
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(news.location ?? output?.location, env.region);
      const previousFactor = olds?.replicationFactor ?? output?.replicationFactor;
      const nextFactor = news.replicationFactor ?? previousFactor;
      return replaceOnIdentity({
        previousId: olds?.topicId ?? output?.topicId,
        nextId: news.topicId ? rfc1035(news.topicId, "topic") : (olds?.topicId ?? output?.topicId),
        previousParent: olds?.cluster ?? output?.cluster,
        nextParent: clusterOf(news.cluster, env.project, location),
        extra:
          previousFactor !== undefined && nextFactor !== undefined && previousFactor !== nextFactor,
      });
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(olds?.location ?? output?.location, env.region);
      const topicId = yield* toPhysicalId(id, olds?.topicId, output?.topicId, "topic");
      const cluster =
        olds?.cluster !== undefined
          ? clusterOf(olds.cluster, env.project, location)
          : (output?.cluster ?? "");
      const name = output?.name ?? (cluster.length > 0 ? resourceName(cluster, topicId) : "");
      const existing = yield* getTopic(name);
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing, env.project);
      return output !== undefined || olds !== undefined ? attrs : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        const clusters = yield* listAlchemyClusters(env.project, env.region);
        const topics = yield* Effect.forEach(
          clusters.filter((cluster) => (cluster.name ?? "").length > 0),
          (cluster: kafka.Cluster) =>
            collectPages(
              kafka.listProjectsLocationsClustersTopics.pages({
                parent: cluster.name!,
                pageSize: 1000,
              }),
              (page) => page.topics,
            ).pipe(Effect.catchTag("NotFound", () => Effect.succeed([] as kafka.Topic[]))),
          { concurrency: 4 },
        );
        return topics.flat().map((topic) => toAttrs(topic, env.project));
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(news.location ?? output?.location, env.region);
      const cluster = clusterOf(news.cluster, env.project, location);
      const topicId = yield* toPhysicalId(id, news.topicId, output?.topicId, "topic");
      const name = output?.name ?? resourceName(cluster, topicId);
      const partitionCount = news.partitionCount ?? DEFAULT_PARTITION_COUNT;
      const replicationFactor = news.replicationFactor ?? DEFAULT_REPLICATION_FACTOR;
      const configs = news.configs;

      let current = yield* getTopic(name);

      if (current === undefined) {
        const created = yield* retryTransient(
          kafka
            .createProjectsLocationsClustersTopics({
              parent: cluster,
              topicId,
              body: {
                partitionCount,
                replicationFactor,
                configs,
              },
            })
            .pipe(Effect.catchTag("Conflict", () => getTopic(name))),
        );
        current = created ?? undefined;
        // A new topic reads back NotFound (UnknownTopicOrPartition) until its
        // partitions are assigned; block until it is readable.
        if (current !== undefined) {
          current = yield* getTopic(name).pipe(
            Effect.repeat({
              schedule: Schedule.spaced("5 seconds"),
              until: (topic) => topic !== undefined,
              times: 24,
            }),
          );
        }
      }

      if (current === undefined) {
        return yield* new ClustersTopicNotResolved({ resourceName: name });
      }

      const partitionsChanged =
        (current.partitionCount ?? DEFAULT_PARTITION_COUNT) !== partitionCount;
      const configsChanged =
        fingerprint(stringMapOf(current.configs)) !== fingerprint(stringMapOf(configs));

      if (partitionsChanged || configsChanged) {
        current = yield* kafka.patchProjectsLocationsClustersTopics({
          name,
          updateMask: fieldMask([
            partitionsChanged && "partition_count",
            configsChanged && "configs",
          ]),
          body: {
            partitionCount,
            configs,
          },
        });
      }

      return toAttrs(current, env.project);
    }),

    delete: Effect.fn(function* ({ output }) {
      yield* kafka
        .deleteProjectsLocationsClustersTopics({ name: output.name })
        .pipe(Effect.catchTag("NotFound", () => Effect.void));
    }),
  });