sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ BigQuery/ Dataset.ts

import * as bigquery from "@distilled.cloud/gcp/bigquery_v2";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { tagRecord } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Stream from "effect/Stream";
import { GcpEnvironment } from "../Environment.ts";
import {
  createInternalLabels,
  diffLabels,
  hasAlchemyLabels,
  stripInternalLabels,
  toLabels,
} from "../Labels.ts";
import type { Providers } from "../Providers.ts";

const MAX_DATASET_ID_LENGTH = 1024;

export type DatasetAccess = bigquery.DatasetAccessItem;

export type DatasetEncryption = {
  /**
   * Cloud KMS key used as the default encryption key for new tables,
   * as `projects/{project}/locations/{location}/keyRings/{keyRing}/cryptoKeys/{cryptoKey}`.
   */
  kmsKeyName?: string;
};

export type DatasetProps = {
  /**
   * Dataset id (the `{datasetId}` segment of
   * `projects/{project}/datasets/{datasetId}`). If omitted, a unique name
   * is generated from the stack, stage, and logical id. Must contain only
   * letters, numbers, and underscores; max 1024 characters. Immutable —
   * changing it replaces the dataset.
   */
  datasetId?: string;
  /**
   * Geographic location (`US`, `EU`, `US-CENTRAL1`, `us-central1`, …).
   * Immutable — changing it replaces the dataset.
   * @default the stack's GCP region (`GCP.Region`, profile region, `us-central1`)
   */
  location?: string;
  /**
   * User labels. Alchemy ownership labels are merged in automatically.
   */
  labels?: Record<string, string>;
  /**
   * User-friendly description of the dataset.
   */
  description?: string;
  /**
   * Descriptive name shown in the console.
   */
  friendlyName?: string;
  /**
   * Default lifetime of newly created tables, in milliseconds. Minimum
   * `"3600000"` (one hour). Set to `"0"` to clear.
   */
  defaultTableExpirationMs?: string;
  /**
   * Default partition expiration for new partitioned tables, in
   * milliseconds. Set to `"0"` to clear.
   */
  defaultPartitionExpirationMs?: string;
  /**
   * Time travel window in hours (`"48"` to `"168"`). Defaults to `"168"`
   * when omitted at create.
   */
  maxTimeTravelHours?: string;
  /**
   * Storage billing model (`LOGICAL` or `PHYSICAL`).
   */
  storageBillingModel?: bigquery.DatasetStorageBillingModelEnum | (string & {});
  /**
   * When true, dataset and table names are case-insensitive.
   * @default false
   */
  isCaseInsensitive?: boolean;
  /**
   * Default collation for new tables (`"und:ci"` or `""`).
   */
  defaultCollation?: string;
  /**
   * Default rounding mode for new tables
   * (`ROUND_HALF_AWAY_FROM_ZERO`, `ROUND_HALF_EVEN`).
   */
  defaultRoundingMode?: bigquery.DatasetDefaultRoundingModeEnum | (string & {});
  /**
   * Default Cloud KMS encryption for newly created tables.
   */
  defaultEncryptionConfiguration?: DatasetEncryption;
  /**
   * Access control entries. If omitted, BigQuery installs its default
   * project-owner/reader/writer grants. Providing this list replaces the
   * entire ACL.
   */
  access?: DatasetAccess[];
  /**
   * Delete all tables, views, and routines before destroying the dataset.
   * @default false
   */
  forceDestroy?: boolean;
};

export type Dataset = Resource<
  "GCP.BigQuery.Dataset",
  DatasetProps,
  {
    /** Resource name `projects/{project}/datasets/{datasetId}`. */
    name: string;
    /** Dataset id. */
    datasetId: string;
    /** Project id. */
    project: string;
    /** Fully-qualified id `projectId:datasetId`. */
    id: string;
    /** Geographic location. */
    location: string;
    /** User labels (Alchemy ownership labels stripped). */
    labels: Record<string, string>;
    /** User-friendly description. */
    description: string | undefined;
    /** Descriptive console name. */
    friendlyName: string | undefined;
    /** Default table expiration, in milliseconds. */
    defaultTableExpirationMs: string | undefined;
    /** Default partition expiration, in milliseconds. */
    defaultPartitionExpirationMs: string | undefined;
    /** Time travel window in hours. */
    maxTimeTravelHours: string | undefined;
    /** Storage billing model. */
    storageBillingModel: string | undefined;
    /** Whether names are case-insensitive. */
    isCaseInsensitive: boolean;
    /** Default collation for new tables. */
    defaultCollation: string | undefined;
    /** Default rounding mode for new tables. */
    defaultRoundingMode: string | undefined;
    /** Default KMS key for new tables, if any. */
    kmsKeyName: string | undefined;
    /** Access control entries currently on the dataset. */
    access: DatasetAccess[];
    /** Dataset type (`DEFAULT`, `LINKED`, `EXTERNAL`, …). */
    type: string | undefined;
    /** Self-link URL. */
    selfLink: string | undefined;
    /** Server etag. */
    etag: string | undefined;
    /** Creation time in milliseconds since epoch. */
    creationTime: string | undefined;
    /** Last-modified time in milliseconds since epoch. */
    lastModifiedTime: string | undefined;
  },
  never,
  Providers
>;

/**
 * A Google BigQuery dataset — a container for tables, views, and routines.
 *
 * Changing `datasetId` or `location` replaces the dataset. Labels,
 * description, expiration, collation, encryption, and access update in
 * place. Destroy fails if the dataset still contains tables unless
 * `forceDestroy` is set.
 *
 * ### Creating a Dataset
 * **Example:** Generated name
 * ```typescript
 * const dataset = yield* GCP.BigQuery.Dataset("Analytics", {});
 * ```
 *
 * **Example:** Explicit id, location, and labels
 * ```typescript
 * const dataset = yield* GCP.BigQuery.Dataset("Analytics", {
 *   datasetId: "app_analytics",
 *   location: "US-CENTRAL1",
 *   labels: { env: "prod" },
 *   description: "application events",
 *   forceDestroy: true,
 * });
 * ```
 *
 * ### Querying
 * **Example:** Run a GoogleSQL query against the dataset
 * ```typescript
 * const query = yield* GCP.BigQuery.Query(dataset);
 * const result = yield* query({ query: "SELECT 1 AS n" });
 * ```
 *
 * @resource
 * @category BigQuery
 */
export const Dataset = Resource<Dataset>("GCP.BigQuery.Dataset");

export class DatasetNotResolved extends Data.TaggedError("GCP.BigQuery.DatasetNotResolved")<{
  resourceName: string;
}> {}

const userLabels = (
  labels: Record<string, string | undefined> | null | undefined,
): Record<string, string> => stripInternalLabels(tagRecord(labels));

const resourceName = (project: string, datasetId: string) =>
  `projects/${project}/datasets/${datasetId}`;

const datasetIdOf = (dataset: bigquery.Dataset | bigquery.DatasetListDatasetsItem) => {
  const fromRef = dataset.datasetReference?.datasetId;
  if (fromRef !== undefined && fromRef.length > 0) return fromRef;
  const id = dataset.id ?? "";
  const colon = id.indexOf(":");
  return colon >= 0 ? id.slice(colon + 1) : id;
};

const projectOf = (
  dataset: bigquery.Dataset | bigquery.DatasetListDatasetsItem,
  fallback: string,
) => {
  const fromRef = dataset.datasetReference?.projectId;
  if (fromRef !== undefined && fromRef.length > 0) return fromRef;
  const id = dataset.id ?? "";
  const colon = id.indexOf(":");
  return colon > 0 ? id.slice(0, colon) : fallback;
};

const toId = (id: string, datasetId: string | undefined, existing?: string) =>
  Effect.gen(function* () {
    if (datasetId !== undefined) return datasetId;
    if (existing !== undefined) return existing;
    const generated = yield* createPhysicalName({
      id,
      maxLength: MAX_DATASET_ID_LENGTH,
      lowercase: true,
      delimiter: "_",
    });
    return generated.replaceAll("-", "_");
  });

const toAttrs = (
  dataset: bigquery.Dataset | bigquery.DatasetListDatasetsItem,
  fallbackProject: string,
  region: string,
) => {
  const datasetId = datasetIdOf(dataset);
  const project = projectOf(dataset, fallbackProject);
  const full = dataset as bigquery.Dataset;
  return {
    name: resourceName(project, datasetId),
    datasetId,
    project,
    id: full.id ?? `${project}:${datasetId}`,
    location: dataset.location ?? region,
    labels: userLabels(dataset.labels),
    description: full.description,
    friendlyName: dataset.friendlyName,
    defaultTableExpirationMs: full.defaultTableExpirationMs,
    defaultPartitionExpirationMs: full.defaultPartitionExpirationMs,
    maxTimeTravelHours: full.maxTimeTravelHours,
    storageBillingModel: full.storageBillingModel,
    isCaseInsensitive: full.isCaseInsensitive === true,
    defaultCollation: full.defaultCollation,
    defaultRoundingMode: full.defaultRoundingMode,
    kmsKeyName: full.defaultEncryptionConfiguration?.kmsKeyName,
    access: full.access ?? [],
    type: dataset.type,
    selfLink: full.selfLink,
    etag: full.etag,
    creationTime: full.creationTime,
    lastModifiedTime: full.lastModifiedTime,
  };
};

const getById = (project: string, datasetId: string) =>
  bigquery
    .getDatasets({ projectId: project, datasetId })
    .pipe(Effect.catchTag("NotFound", () => Effect.undefined));

const sameString = (left: string | undefined, right: string | undefined) =>
  (left ?? "") === (right ?? "");

const accessJson = (access: readonly DatasetAccess[] | undefined) => JSON.stringify(access ?? []);

export const DatasetProvider = () =>
  Provider.succeed(Dataset, {
    stables: ["name", "datasetId", "project", "id", "location", "creationTime"],

    diff: Effect.fn(function* ({ news, olds, output }) {
      if (!isResolved(news)) return undefined;
      const previousId = olds?.datasetId ?? output?.datasetId;
      if (
        previousId !== undefined &&
        news.datasetId !== undefined &&
        news.datasetId !== previousId
      ) {
        return { action: "replace" as const, deleteFirst: false };
      }
      const previousLocation = olds?.location ?? output?.location;
      const nextLocation = news.location;
      if (
        previousLocation !== undefined &&
        nextLocation !== undefined &&
        previousLocation.toUpperCase() !== nextLocation.toUpperCase()
      ) {
        const nextId = news.datasetId ?? previousId;
        return {
          action: "replace" as const,
          deleteFirst: nextId !== undefined && nextId === previousId,
        };
      }
      return undefined;
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const datasetId = yield* toId(id, olds?.datasetId, output?.datasetId);
      const project = output?.project ?? env.project;
      const existing = yield* getById(project, datasetId);
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing, project, env.region);
      return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        return yield* bigquery.listDatasets
          .pages({
            projectId: env.project,
            maxResults: 1000,
            all: true,
          })
          .pipe(
            Stream.flatMap((page) => Stream.fromIterable(page.datasets ?? [])),
            Stream.filter((dataset) =>
              Object.keys(dataset.labels ?? {}).some((key) => key.startsWith("alchemy-")),
            ),
            Stream.map((dataset) => toAttrs(dataset, env.project, env.region)),
            Stream.runCollect,
            Effect.map((chunk) => Array.from(chunk)),
          );
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const datasetId = yield* toId(id, news.datasetId, output?.datasetId);
      const project = output?.project ?? env.project;
      const location = news.location ?? output?.location ?? env.region;
      const desiredLabels = {
        ...toLabels(news.labels),
        ...(yield* createInternalLabels(id)),
      };
      const name = resourceName(project, datasetId);

      let current = yield* getById(project, datasetId);

      if (current === undefined) {
        const created = yield* bigquery
          .insertDatasets({
            projectId: project,
            body: {
              datasetReference: { projectId: project, datasetId },
              location,
              labels: desiredLabels,
              description: news.description,
              friendlyName: news.friendlyName,
              defaultTableExpirationMs: news.defaultTableExpirationMs,
              defaultPartitionExpirationMs: news.defaultPartitionExpirationMs,
              maxTimeTravelHours: news.maxTimeTravelHours,
              storageBillingModel: news.storageBillingModel,
              isCaseInsensitive: news.isCaseInsensitive === true ? true : undefined,
              defaultCollation: news.defaultCollation,
              defaultRoundingMode: news.defaultRoundingMode,
              defaultEncryptionConfiguration: news.defaultEncryptionConfiguration,
              access: news.access,
            },
          })
          .pipe(Effect.catchTag("Conflict", () => getById(project, datasetId)));
        current = created ?? undefined;
      }

      if (current === undefined) {
        return yield* new DatasetNotResolved({ resourceName: name });
      }

      const observedLabels = tagRecord(current.labels);
      const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
      if (removed.length > 0) {
        // PATCH merges labels; removing one needs a JSON null the typed body
        // cannot carry, so replace the labels with PUT (get-modify-put).
        current = yield* bigquery.updateDatasets({
          projectId: project,
          datasetId,
          body: { ...current, labels: desiredLabels },
        });
      }
      const labelsChanged = removed.length === 0 && upsert.length > 0;
      const descriptionChanged = !sameString(current.description, news.description);
      const friendlyNameChanged = !sameString(current.friendlyName, news.friendlyName);
      const tableExpirationChanged =
        news.defaultTableExpirationMs !== undefined &&
        !sameString(current.defaultTableExpirationMs, news.defaultTableExpirationMs);
      const partitionExpirationChanged =
        news.defaultPartitionExpirationMs !== undefined &&
        !sameString(current.defaultPartitionExpirationMs, news.defaultPartitionExpirationMs);
      const timeTravelChanged =
        news.maxTimeTravelHours !== undefined &&
        !sameString(current.maxTimeTravelHours, news.maxTimeTravelHours);
      const billingChanged =
        news.storageBillingModel !== undefined &&
        (current.storageBillingModel ?? "") !== news.storageBillingModel;
      const caseChanged =
        (current.isCaseInsensitive === true) !== (news.isCaseInsensitive === true);
      const collationChanged =
        news.defaultCollation !== undefined &&
        !sameString(current.defaultCollation, news.defaultCollation);
      const roundingChanged =
        news.defaultRoundingMode !== undefined &&
        (current.defaultRoundingMode ?? "") !== news.defaultRoundingMode;
      const encryptionChanged =
        news.defaultEncryptionConfiguration !== undefined &&
        (current.defaultEncryptionConfiguration?.kmsKeyName ?? "") !==
          (news.defaultEncryptionConfiguration.kmsKeyName ?? "");
      const accessChanged =
        news.access !== undefined && accessJson(current.access) !== accessJson(news.access);

      const metadataChanged =
        labelsChanged ||
        descriptionChanged ||
        friendlyNameChanged ||
        tableExpirationChanged ||
        partitionExpirationChanged ||
        timeTravelChanged ||
        billingChanged ||
        caseChanged ||
        collationChanged ||
        roundingChanged ||
        encryptionChanged;

      if (metadataChanged || accessChanged) {
        const body: bigquery.Dataset = {};
        if (labelsChanged) {
          body.labels = desiredLabels;
        }
        if (descriptionChanged) {
          body.description = news.description ?? "";
        }
        if (friendlyNameChanged) {
          body.friendlyName = news.friendlyName ?? "";
        }
        if (tableExpirationChanged) {
          body.defaultTableExpirationMs = news.defaultTableExpirationMs;
        }
        if (partitionExpirationChanged) {
          body.defaultPartitionExpirationMs = news.defaultPartitionExpirationMs;
        }
        if (timeTravelChanged) {
          body.maxTimeTravelHours = news.maxTimeTravelHours;
        }
        if (billingChanged) {
          body.storageBillingModel = news.storageBillingModel;
        }
        if (caseChanged) {
          body.isCaseInsensitive = news.isCaseInsensitive === true;
        }
        if (collationChanged) {
          body.defaultCollation = news.defaultCollation;
        }
        if (roundingChanged) {
          body.defaultRoundingMode = news.defaultRoundingMode;
        }
        if (encryptionChanged) {
          body.defaultEncryptionConfiguration = news.defaultEncryptionConfiguration;
        }
        if (accessChanged) {
          body.access = news.access;
        }
        current = yield* bigquery.patchDatasets({
          projectId: project,
          datasetId,
          updateMode:
            accessChanged && metadataChanged
              ? "UPDATE_FULL"
              : accessChanged
                ? "UPDATE_ACL"
                : "UPDATE_METADATA",
          body,
        });
      }

      return toAttrs(current, project, env.region);
    }),

    delete: Effect.fn(function* ({ olds, output, force }) {
      yield* bigquery
        .deleteDatasets({
          projectId: output.project,
          datasetId: output.datasetId,
          deleteContents: olds.forceDestroy === true || force === true,
        })
        .pipe(Effect.catchTag("NotFound", () => Effect.void));
    }),
  });