sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ Dataplex/ DataScan.ts

import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { tagRecord } from "alchemy/Tags";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import { GcpEnvironment } from "../Environment.ts";
import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
import type { Providers } from "../Providers.ts";
import {
  DataplexNotResolved,
  collectPages,
  fingerprint,
  hasAlchemyLabelMap,
  listAtLocation,
  normalizeLocation,
  parentOf,
  parseName,
  replaceOnIdentity,
  retryQuota,
  toPhysicalId,
  userLabels,
  waitForOperation,
  waitUntilExists,
  waitUntilGone,
} from "./internal.ts";

export type DataScanDataSource = dataplex.GoogleCloudDataplexV1DataSource;
export type DataScanExecutionSpec = dataplex.GoogleCloudDataplexV1DataScanExecutionSpec;
export type DataScanExecutionIdentity = dataplex.GoogleCloudDataplexV1ExecutionIdentity;
export type DataQualitySpec = dataplex.GoogleCloudDataplexV1DataQualitySpec;
export type DataProfileSpec = dataplex.GoogleCloudDataplexV1DataProfileSpec;
export type DataDiscoverySpec = dataplex.GoogleCloudDataplexV1DataDiscoverySpec;
export type DataDocumentationSpec = dataplex.GoogleCloudDataplexV1DataDocumentationSpec;
export type UnstructuredDataProfileSpec = dataplex.GoogleCloudDataplexV1UnstructuredDataProfileSpec;

export type DataScanProps = {
  /**
   * DataScan id. If omitted, a unique name is generated. Must contain
   * only lowercase letters, numbers and hyphens, start with a letter,
   * end with a letter or number, and be 1-63 characters. Immutable —
   * changing it replaces the scan.
   */
  dataScanId?: string;
  /**
   * Region (`us-central1`, …). Immutable — changing it replaces the
   * scan.
   * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
   */
  location?: string;
  /**
   * Description (1-1024 characters).
   */
  description?: string;
  /**
   * User-friendly display name (1-256 characters).
   */
  displayName?: string;
  /**
   * User labels. Alchemy ownership labels are merged in automatically.
   */
  labels?: Record<string, string>;
  /**
   * Immutable data source (Dataplex entity or BigQuery / GCS resource).
   * Changing it replaces the scan.
   */
  data: DataScanDataSource;
  /**
   * Execution settings. Defaults to on-demand.
   */
  executionSpec?: DataScanExecutionSpec;
  /**
   * Immutable identity used to run the scan. Changing it replaces the
   * scan.
   */
  executionIdentity?: DataScanExecutionIdentity;
  /**
   * Data quality scan settings. Mutually exclusive with the other spec
   * fields.
   */
  dataQualitySpec?: DataQualitySpec;
  /**
   * Data profile scan settings.
   */
  dataProfileSpec?: DataProfileSpec;
  /**
   * Data discovery scan settings.
   */
  dataDiscoverySpec?: DataDiscoverySpec;
  /**
   * Data documentation scan settings.
   */
  dataDocumentationSpec?: DataDocumentationSpec;
  /**
   * Unstructured data profile scan settings.
   */
  unstructuredDataProfileSpec?: UnstructuredDataProfileSpec;
};

export type DataScan = Resource<
  "GCP.Dataplex.DataScan",
  DataScanProps,
  {
    /** Full resource name. */
    name: string;
    /** DataScan id (last path segment). */
    dataScanId: string;
    /** Project id. */
    project: string;
    /** Location id. */
    location: string;
    /** Description. */
    description: string | undefined;
    /** Display name. */
    displayName: string | undefined;
    /** User labels (Alchemy ownership labels stripped). */
    labels: Record<string, string>;
    /** Current state. */
    state: string | undefined;
    /** Data source. */
    data: DataScanDataSource | undefined;
    /** Execution spec. */
    executionSpec: DataScanExecutionSpec | undefined;
    /** Scan type. */
    type: string | undefined;
    /** Data quality spec, if any. */
    dataQualitySpec: DataQualitySpec | undefined;
    /** Data profile spec, if any. */
    dataProfileSpec: DataProfileSpec | undefined;
    /** Data discovery spec, if any. */
    dataDiscoverySpec: DataDiscoverySpec | undefined;
    /** Data documentation spec, if any. */
    dataDocumentationSpec: DataDocumentationSpec | undefined;
    /** Unstructured data profile spec, if any. */
    unstructuredDataProfileSpec: UnstructuredDataProfileSpec | undefined;
    /** System uid. */
    uid: string | undefined;
    /** RFC3339 creation timestamp. */
    createTime: string | undefined;
    /** RFC3339 last-update timestamp. */
    updateTime: string | undefined;
  },
  never,
  Providers
>;

/**
 * A Dataplex DataScan that profiles, quality-checks, discovers, or
 * documents a BigQuery table, dataset, or Cloud Storage bucket.
 *
 * Location, scan id, data source, and execution identity are immutable.
 * Description, display name, labels, execution spec, and scan specs
 * update in place.
 *
 * ### Creating a DataScan
 * **Example:** On-demand profile scan of a BigQuery table
 * ```typescript
 * const scan = yield* GCP.Dataplex.DataScan("OrdersProfile", {
 *   displayName: "orders profile",
 *   labels: { env: "test" },
 *   data: {
 *     resource:
 *       "//bigquery.googleapis.com/projects/my-project/datasets/sales/tables/orders",
 *   },
 *   dataProfileSpec: {},
 * });
 * ```
 *
 * @resource
 * @category Dataplex
 */
export const DataScan = Resource<DataScan>("GCP.Dataplex.DataScan");

const resourceName = (project: string, location: string, dataScanId: string) =>
  `projects/${project}/locations/${location}/dataScans/${dataScanId}`;

const toAttrs = (scan: dataplex.GoogleCloudDataplexV1DataScan, project: string) => {
  const name = scan.name ?? "";
  const parsed = parseName(name, "dataScans");
  return {
    name,
    dataScanId: parsed.id,
    project: parsed.project || project,
    location: parsed.location,
    description: scan.description,
    displayName: scan.displayName,
    labels: userLabels(scan.labels),
    state: scan.state,
    data: scan.data,
    executionSpec: scan.executionSpec,
    type: scan.type,
    dataQualitySpec: scan.dataQualitySpec,
    dataProfileSpec: scan.dataProfileSpec,
    dataDiscoverySpec: scan.dataDiscoverySpec,
    dataDocumentationSpec: scan.dataDocumentationSpec,
    unstructuredDataProfileSpec: scan.unstructuredDataProfileSpec,
    uid: scan.uid,
    createTime: scan.createTime,
    updateTime: scan.updateTime,
  };
};

const getByName = (name: string) =>
  retryQuota(dataplex.getProjectsLocationsDataScans({ name, view: "FULL" })).pipe(
    Effect.catchTag("NotFound", () => Effect.undefined),
  );

const listScans = (project: string, region: string) => {
  const collect = (parent: string) =>
    collectPages(
      dataplex.listProjectsLocationsDataScans.pages({
        parent,
        pageSize: 1000,
      }),
      (page) => page.dataScans,
    ).pipe(
      Effect.map((items) => items.filter((item) => hasAlchemyLabelMap(item.labels))),
      Effect.catchTag("NotFound", () => Effect.succeed([])),
    );
  return listAtLocation(project, region, collect);
};

const dataSourceOf = (data: DataScanDataSource | undefined) => ({
  entity: data?.entity ?? "",
  resource: data?.resource ?? "",
});

export const DataScanProvider = () =>
  Provider.succeed(DataScan, {
    stables: ["name", "dataScanId", "project", "location", "uid", "createTime"],

    diff: Effect.fn(function* ({ news, olds, output }) {
      if (!isResolved(news)) return undefined;
      const env = yield* GcpEnvironment.current;
      return replaceOnIdentity({
        previousId: olds?.dataScanId ?? output?.dataScanId,
        nextId: news.dataScanId ?? olds?.dataScanId ?? output?.dataScanId,
        previousLocation: normalizeLocation(olds?.location ?? output?.location, env.region),
        nextLocation: normalizeLocation(
          news.location ?? olds?.location ?? output?.location,
          env.region,
        ),
        extra:
          fingerprint(dataSourceOf(news.data)) !==
            fingerprint(dataSourceOf(olds?.data ?? output?.data)) ||
          fingerprint(news.executionIdentity ?? olds?.executionIdentity) !==
            fingerprint(olds?.executionIdentity),
      });
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const dataScanId = yield* toPhysicalId(id, olds?.dataScanId, output?.dataScanId, "datascan");
      const location = normalizeLocation(olds?.location ?? output?.location, env.region);
      const name = output?.name ?? resourceName(env.project, location, dataScanId);
      const existing = yield* getByName(name);
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing, env.project);
      return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        const items = yield* listScans(env.project, env.region);
        return items.map((item) => toAttrs(item, env.project));
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const dataScanId = yield* toPhysicalId(id, news.dataScanId, output?.dataScanId, "datascan");
      const location = normalizeLocation(news.location ?? output?.location, env.region);
      const name = resourceName(env.project, location, dataScanId);
      const desiredLabels = {
        ...toLabels(news.labels),
        ...(yield* createInternalLabels(id)),
      };

      let current = yield* getByName(output?.name ?? name);

      if (current === undefined) {
        const created = yield* retryQuota(
          dataplex.createProjectsLocationsDataScans({
            parent: parentOf(env.project, location),
            dataScanId,
            body: {
              description: news.description,
              displayName: news.displayName,
              labels: desiredLabels,
              data: news.data,
              executionSpec: news.executionSpec,
              executionIdentity: news.executionIdentity,
              dataQualitySpec: news.dataQualitySpec,
              dataProfileSpec: news.dataProfileSpec,
              dataDiscoverySpec: news.dataDiscoverySpec,
              dataDocumentationSpec: news.dataDocumentationSpec,
              unstructuredDataProfileSpec: news.unstructuredDataProfileSpec,
            },
          }),
        ).pipe(Effect.catchTag("Conflict", () => Effect.undefined));
        if (created !== undefined) {
          yield* waitForOperation(created);
        }
        current = yield* waitUntilExists(getByName(name), name);
      }

      if (current === undefined) {
        return yield* new DataplexNotResolved({ resourceName: name });
      }

      const observedLabels = tagRecord(current.labels);
      const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
      const labelsChanged = upsert.length > 0 || removed.length > 0;
      const descriptionChanged = (current.description ?? "") !== (news.description ?? "");
      const displayNameChanged = (current.displayName ?? "") !== (news.displayName ?? "");
      const executionChanged =
        fingerprint(current.executionSpec) !== fingerprint(news.executionSpec);
      const qualityChanged =
        fingerprint(current.dataQualitySpec) !== fingerprint(news.dataQualitySpec);
      const profileChanged =
        fingerprint(current.dataProfileSpec) !== fingerprint(news.dataProfileSpec);
      const discoveryChanged =
        fingerprint(current.dataDiscoverySpec) !== fingerprint(news.dataDiscoverySpec);
      const documentationChanged =
        fingerprint(current.dataDocumentationSpec) !== fingerprint(news.dataDocumentationSpec);
      const unstructuredChanged =
        fingerprint(current.unstructuredDataProfileSpec) !==
        fingerprint(news.unstructuredDataProfileSpec);

      if (
        labelsChanged ||
        descriptionChanged ||
        displayNameChanged ||
        executionChanged ||
        qualityChanged ||
        profileChanged ||
        discoveryChanged ||
        documentationChanged ||
        unstructuredChanged
      ) {
        const operation = yield* retryQuota(
          dataplex.patchProjectsLocationsDataScans({
            name: current.name ?? name,
            updateMask: [
              labelsChanged ? "labels" : undefined,
              descriptionChanged ? "description" : undefined,
              displayNameChanged ? "displayName" : undefined,
              executionChanged ? "executionSpec" : undefined,
              qualityChanged ? "dataQualitySpec" : undefined,
              profileChanged ? "dataProfileSpec" : undefined,
              discoveryChanged ? "dataDiscoverySpec" : undefined,
              documentationChanged ? "dataDocumentationSpec" : undefined,
              unstructuredChanged ? "unstructuredDataProfileSpec" : undefined,
            ]
              .filter((field): field is string => field !== undefined)
              .join(","),
            body: {
              name: current.name ?? name,
              description: news.description,
              displayName: news.displayName,
              labels: desiredLabels,
              executionSpec: news.executionSpec,
              dataQualitySpec: news.dataQualitySpec,
              dataProfileSpec: news.dataProfileSpec,
              dataDiscoverySpec: news.dataDiscoverySpec,
              dataDocumentationSpec: news.dataDocumentationSpec,
              unstructuredDataProfileSpec: news.unstructuredDataProfileSpec,
            },
          }),
        );
        yield* waitForOperation(operation);
        current = yield* waitUntilExists(getByName(current.name ?? name), current.name ?? name);
      }

      return toAttrs(current, env.project);
    }),

    delete: Effect.fn(function* ({ output }) {
      const operation = yield* retryQuota(
        dataplex.deleteProjectsLocationsDataScans({
          name: output.name,
          force: true,
        }),
      ).pipe(
        Effect.retry({
          while: (error) => error._tag === "Conflict",
          times: 8,
          schedule: Schedule.spaced("2 seconds"),
        }),
        Effect.catchTag("NotFound", () => Effect.undefined),
      );
      if (operation !== undefined) {
        yield* waitForOperation(operation, { notFoundOk: true });
      }
      yield* waitUntilGone(getByName(output.name), output.name);
    }),
  });