sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ BigLake/ CatalogsDatabase.ts

import * as biglake from "@distilled.cloud/gcp/biglake_v1";
import { unownedUnlessPrevious } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { tagRecord } from "alchemy/Tags";
import * as Effect from "effect/Effect";
import { GcpEnvironment } from "../Environment.ts";
import type { Providers } from "../Providers.ts";
import {
  BiglakeNotResolved,
  listLocations,
  createInternalLabels,
  expandCatalog,
  hasAlchemyLabelMap,
  hasAlchemyLabels,
  ignoreGone,
  listCatalogs,
  listChildResources,
  listDatabases,
  listTables,
  locationParent,
  mergeParameters,
  missingGet,
  namedOf,
  normalizeLocation,
  parseResourceName,
  replaceOnIdentity,
  retryTransient,
  sameJson,
  sameText,
  toPhysicalId,
  updateMaskOf,
  userLabels,
  waitUntilGone,
} from "./internal.ts";

export type HiveDatabaseOptions = {
  /**
   * Cloud Storage folder URI where the database data is stored, starting
   * with `gs://`.
   */
  locationUri?: string;
  /**
   * User-supplied Hive database parameters. Alchemy ownership keys
   * (`alchemy-stack`, `alchemy-stage`, `alchemy-id`) are merged in
   * automatically — BigLake databases have no labels field.
   */
  parameters?: Record<string, string>;
};

export type CatalogsDatabaseProps = {
  /**
   * Parent catalog. Full name
   * `projects/{project}/locations/{location}/catalogs/{catalog}` or the
   * catalog id (combined with `location`). Immutable — changing it
   * replaces the database.
   */
  catalog: string;
  /**
   * Database id (the `{database}` segment). If omitted, a unique id is
   * generated. Immutable — changing it replaces the database.
   */
  databaseId?: string;
  /**
   * Location used when `catalog` is a bare id.
   * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
   */
  location?: string;
  /**
   * Database type.
   * @default "HIVE"
   */
  type?: biglake.DatabaseTypeEnum | (string & {});
  /**
   * Hive database options. Required when `type` is `HIVE`.
   */
  hiveOptions?: HiveDatabaseOptions;
  /**
   * Shortcut for `hiveOptions.locationUri`.
   */
  locationUri?: string;
  /**
   * Shortcut for `hiveOptions.parameters`. Alchemy ownership keys are
   * merged in automatically.
   */
  parameters?: Record<string, string>;
};

export type CatalogsDatabase = Resource<
  "GCP.BigLake.CatalogsDatabase",
  CatalogsDatabaseProps,
  {
    /** Full resource name. */
    name: string;
    /** Database id (last path segment). */
    databaseId: string;
    /** Parent catalog resource name. */
    catalog: string;
    /** Project id. */
    project: string;
    /** Location id. */
    location: string;
    /** Database type. */
    type: string | undefined;
    /** Hive location URI. */
    locationUri: string | undefined;
    /** User Hive parameters (Alchemy ownership keys stripped). */
    parameters: Record<string, string>;
    /** RFC3339 creation timestamp. */
    createTime: string | undefined;
    /** RFC3339 last-update timestamp. */
    updateTime: string | undefined;
    /** RFC3339 deletion timestamp, if soft-deleted. */
    deleteTime: string | undefined;
    /** RFC3339 expire timestamp after delete. */
    expireTime: string | undefined;
  },
  never,
  Providers
>;

/**
 * A Hive database inside a BigLake Metastore catalog.
 *
 * Databases have no labels field — Alchemy stamps ownership into
 * `hiveOptions.parameters` so `list` / nuke can find them. Catalog,
 * database id, and type are immutable. Hive location URI and parameters
 * update in place.
 *
 * ### Creating a Database
 * **Example:** Hive database on a GCS prefix
 * ```typescript
 * const catalog = yield* GCP.BigLake.Catalog("Lake", {});
 * const database = yield* GCP.BigLake.CatalogsDatabase("Warehouse", {
 *   catalog: catalog.name,
 *   locationUri: `gs://${bucket.bucketName}/warehouse`,
 *   parameters: { owner: "analytics" },
 * });
 * ```
 *
 * **Example:** Named database
 * ```typescript
 * const database = yield* GCP.BigLake.CatalogsDatabase("Warehouse", {
 *   catalog: catalog.name,
 *   databaseId: "sales",
 *   type: "HIVE",
 *   hiveOptions: {
 *     locationUri: "gs://my-bucket/sales",
 *   },
 * });
 * ```
 *
 * @resource
 * @category BigLake
 */
export const CatalogsDatabase = Resource<CatalogsDatabase>("GCP.BigLake.CatalogsDatabase");

const catalogOf = (catalog: string, project: string, location: string) =>
  expandCatalog(catalog, project, location);

const resourceName = (catalog: string, databaseId: string) => `${catalog}/databases/${databaseId}`;

const parametersOf = (database: biglake.Database) => userLabels(database.hiveOptions?.parameters);

const toAttrs = (database: biglake.Database, project: string) => {
  const name = database.name ?? "";
  const parsed = parseResourceName(name, "databases");
  return {
    name,
    databaseId: parsed.id,
    catalog: parsed.parent,
    project,
    location: parsed.location,
    type: database.type,
    locationUri: database.hiveOptions?.locationUri,
    parameters: parametersOf(database),
    createTime: database.createTime,
    updateTime: database.updateTime,
    deleteTime: database.deleteTime,
    expireTime: database.expireTime,
  };
};

const getByName = missingGet(biglake.getProjectsLocationsCatalogsDatabases);

const desiredHiveOptions = (
  news: CatalogsDatabaseProps,
  parameters: Record<string, string>,
): biglake.HiveDatabaseOptions => ({
  locationUri: news.hiveOptions?.locationUri ?? news.locationUri,
  parameters,
});

const emptyAndDeleteDatabase = (name: string) =>
  Effect.gen(function* () {
    const tables = yield* listTables(name);
    yield* Effect.forEach(
      namedOf(tables),
      (table) =>
        ignoreGone(
          biglake.deleteProjectsLocationsCatalogsDatabasesTables({
            name: table.name!,
          }),
        ),
      { concurrency: 4 },
    );
    yield* ignoreGone(biglake.deleteProjectsLocationsCatalogsDatabases({ name }));
  });

export const CatalogsDatabaseProvider = () =>
  Provider.succeed(CatalogsDatabase, {
    stables: ["name", "databaseId", "catalog", "project", "location", "createTime"],

    diff: Effect.fn(function* ({ news, olds, output }) {
      if (!isResolved(news)) return undefined;
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(
        news.location ?? olds?.location ?? output?.location,
        env.region,
      );
      const typeChanged =
        (olds?.type ?? output?.type) !== undefined &&
        (news.type ?? "HIVE") !== (olds?.type ?? output?.type);
      if (typeChanged) {
        return { action: "replace" as const, deleteFirst: true };
      }
      return replaceOnIdentity({
        previousId: olds?.databaseId ?? output?.databaseId,
        nextId: news.databaseId,
        previousLocation: normalizeLocation(olds?.location ?? output?.location, env.region),
        nextLocation: location,
        previousParent: olds?.catalog ?? output?.catalog,
        nextParent: catalogOf(news.catalog, env.project, location),
      });
    }),

    read: Effect.fn(function* ({ id, olds, output, previous }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(olds?.location ?? output?.location, env.region);
      const databaseId = yield* toPhysicalId(id, olds?.databaseId, output?.databaseId);
      const catalog =
        olds?.catalog !== undefined
          ? catalogOf(olds.catalog, env.project, location)
          : (output?.catalog ?? "");
      const name = output?.name ?? (catalog.length > 0 ? resourceName(catalog, databaseId) : "");
      const existing = yield* getByName(name);
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing, env.project);
      return (yield* hasAlchemyLabels(id, tagRecord(existing.hiveOptions?.parameters)))
        ? attrs
        : unownedUnlessPrevious(attrs, previous, "name");
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        const catalogs = (yield* Effect.forEach(listLocations(env.region), (location) =>
          listCatalogs(locationParent(env.project, location)),
        )).flat();
        const databases = yield* listChildResources(namedOf(catalogs), listDatabases);
        return databases
          .filter((item) => hasAlchemyLabelMap(item.hiveOptions?.parameters))
          .map((item) => toAttrs(item, env.project));
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(news.location ?? output?.location, env.region);
      const catalog = catalogOf(news.catalog, env.project, location);
      const databaseId = yield* toPhysicalId(id, news.databaseId, output?.databaseId);
      const name = output?.name ?? resourceName(catalog, databaseId);
      const type = news.type ?? "HIVE";
      const parameters = mergeParameters(
        news.parameters ?? news.hiveOptions?.parameters,
        yield* createInternalLabels(id),
      );
      const hiveOptions = desiredHiveOptions(news, parameters);
      const body: biglake.Database = { type, hiveOptions };

      let current = yield* getByName(name);

      if (current === undefined) {
        const created = yield* retryTransient(
          biglake.createProjectsLocationsCatalogsDatabases({
            parent: catalog,
            databaseId,
            body,
          }),
        ).pipe(Effect.catchTag("Conflict", () => getByName(name)));
        current = created ?? undefined;
      }

      if (current === undefined) {
        return yield* new BiglakeNotResolved({ resourceName: name });
      }

      const currentName = current.name ?? name;
      const typeChanged = !sameText(current.type, type);
      const hiveChanged = !sameJson(current.hiveOptions, hiveOptions);

      if (typeChanged || hiveChanged) {
        current = yield* retryTransient(
          biglake.patchProjectsLocationsCatalogsDatabases({
            name: currentName,
            updateMask: updateMaskOf(
              typeChanged ? "type" : undefined,
              hiveChanged ? "hiveOptions" : undefined,
            ),
            body: { name: currentName, ...body },
          }),
        );
      }

      return toAttrs(current, env.project);
    }),

    delete: Effect.fn(function* ({ output }) {
      if (!output.name) return;
      yield* emptyAndDeleteDatabase(output.name);
      yield* waitUntilGone(getByName(output.name));
    }),
  });