sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ DataLineage/ Process.ts

import * as datalineage from "@distilled.cloud/gcp/datalineage_v1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import { GcpEnvironment } from "../Environment.ts";
import type { Providers } from "../Providers.ts";
import {
  createOwnership,
  desiredAttributes,
  findOwnedProcess,
  listOwnedProcesses,
  locationParent,
  normalizeLocation,
  ownedByAlchemy,
  parseName,
  replaceOnIdentity,
  sameJson,
  sameText,
  toPhysicalId,
  userAttributes,
  waitForOperation,
  waitUntilGone,
} from "./internal.ts";

export type ProcessOrigin = {
  /**
   * Origin system. Use `CUSTOM` unless reporting lineage on behalf of a
   * Google system. Non-`CUSTOM` sources may be restricted and billed.
   * @default "CUSTOM"
   */
  sourceType?: datalineage.GoogleCloudDatacatalogLineageV1OriginSourceTypeEnum | (string & {});
  /**
   * Origin name. For `CUSTOM`, any identifier. For Google systems, a
   * resource name in the same project and location.
   */
  name?: string;
};

export type ProcessProps = {
  /**
   * Process id (the `{process}` segment of
   * `projects/{project}/locations/{location}/processes/{process}`). If
   * omitted, a unique name is generated from the stack, stage, and logical
   * id. Max 200 characters: letters, digits, underscores, hyphens, colons,
   * periods. Immutable — changing it replaces the process.
   */
  processId?: string;
  /**
   * Region (`us-central1`, …). Immutable — changing it replaces the
   * process. `US-CENTRAL1` is accepted and normalized to `us-central1`.
   * @default the stack's GCP region (`GCP.Region`, profile region, `us-central1`)
   */
  location?: string;
  /**
   * Human-readable name shown in UIs. Max 200 characters.
   */
  displayName?: string;
  /**
   * Origin of this process and its runs. Defaults to `{ sourceType:
   * "CUSTOM", name: "alchemy" }`.
   */
  origin?: ProcessOrigin;
  /**
   * Non-semantic attributes (classifying or labeling the process). Max
   * 100 entries. Processes have no labels field, so Alchemy stamps
   * `alchemy-stack` / `alchemy-stage` / `alchemy-id` here for `list` /
   * nuke and strips them from attributes.
   */
  attributes?: Record<string, unknown>;
};

export type Process = Resource<
  "GCP.DataLineage.Process",
  ProcessProps,
  {
    /** Full resource name `projects/{project}/locations/{location}/processes/{process}`. */
    name: string;
    /** Process id (last path segment). */
    processId: string;
    /** Project id. */
    project: string;
    /** Location id. */
    location: string;
    /** Human-readable display name. */
    displayName: string | undefined;
    /** Origin of the process. */
    origin: ProcessOrigin | undefined;
    /** User attributes (Alchemy ownership keys stripped). */
    attributes: Record<string, unknown>;
  },
  never,
  Providers
>;

/**
 * A Data Lineage process — the definition of a data transformation.
 *
 * Processes have no labels field, so Alchemy stamps ownership into
 * `attributes` for `list` / nuke. `processId` and `location` are identity
 * — changing either replaces the process. Display name, origin, and
 * attributes update in place.
 *
 * ### Creating a Process
 * **Example:** Generated name
 * ```typescript
 * const process = yield* GCP.DataLineage.Process("Etl", {
 *   displayName: "nightly etl",
 *   attributes: { env: "dev" },
 * });
 * ```
 *
 * **Example:** Named process with a custom origin
 * ```typescript
 * const process = yield* GCP.DataLineage.Process("Etl", {
 *   processId: "orders-etl",
 *   location: "us-central1",
 *   displayName: "orders etl",
 *   origin: { sourceType: "CUSTOM", name: "dbt" },
 *   attributes: { env: "prod" },
 * });
 * ```
 *
 * @resource
 * @category DataLineage
 */
export const Process = Resource<Process>("GCP.DataLineage.Process");

export class ProcessNotResolved extends Data.TaggedError("GCP.DataLineage.ProcessNotResolved")<{
  resourceName: string;
}> {}

const resourceName = (project: string, location: string, processId: string) =>
  `${locationParent(project, location)}/processes/${processId}`;

const defaultOrigin = (
  origin: ProcessOrigin | undefined,
): datalineage.GoogleCloudDatacatalogLineageV1Origin => ({
  sourceType: origin?.sourceType ?? "CUSTOM",
  name: origin?.name ?? "alchemy",
});

const toAttrs = (
  process: datalineage.GoogleCloudDatacatalogLineageV1Process,
  project: string,
  region: string,
) => {
  const name = process.name ?? "";
  const parsed = parseName(name, "processes", region);
  return {
    name,
    processId: parsed.id,
    project: parsed.project || project,
    location: parsed.location,
    displayName: process.displayName,
    origin: process.origin
      ? {
          sourceType: process.origin.sourceType,
          name: process.origin.name,
        }
      : undefined,
    attributes: userAttributes(process.attributes),
  };
};

const getByName = (name: string) =>
  name.length === 0
    ? Effect.undefined
    : datalineage
        .getProjectsLocationsProcesses({ name })
        .pipe(Effect.catchTag("NotFound", () => Effect.undefined));

export const ProcessProvider = () =>
  Provider.succeed(Process, {
    stables: ["name", "processId", "project", "location"],

    diff: Effect.fn(function* ({ news, olds, output }) {
      if (!isResolved(news)) return undefined;
      const env = yield* GcpEnvironment.current;
      const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
      const nextLocation = normalizeLocation(
        news.location ?? olds?.location ?? output?.location,
        env.region,
      );
      return replaceOnIdentity({
        previousId: olds?.processId ?? output?.processId,
        nextId: news.processId,
        extra: output !== undefined && previousLocation !== nextLocation,
      });
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(olds?.location ?? output?.location, env.region);
      const processId = yield* toPhysicalId(id, olds?.processId, output?.processId);
      const name = output?.name ?? resourceName(env.project, location, processId);
      const existing =
        (yield* getByName(name)) ?? (yield* findOwnedProcess(id, env.project, location));
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing, env.project, env.region);
      return (yield* ownedByAlchemy(id, existing.attributes)) ? attrs : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        const rows = yield* listOwnedProcesses(env.project, env.region);
        return rows.map((row) => toAttrs(row, env.project, env.region));
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(news.location ?? output?.location, env.region);
      const processId = yield* toPhysicalId(id, news.processId, output?.processId);
      const name = output?.name ?? resourceName(env.project, location, processId);
      const ownership = yield* createOwnership(id);
      const attributes = desiredAttributes(news.attributes, ownership);

      let current =
        (yield* getByName(name)) ?? (yield* findOwnedProcess(id, env.project, location));

      if (current === undefined) {
        const created = yield* datalineage
          .createProjectsLocationsProcesses({
            parent: locationParent(env.project, location),
            body: {
              name,
              displayName: news.displayName,
              origin: defaultOrigin(news.origin),
              attributes,
            },
          })
          .pipe(
            Effect.catchTag("Conflict", () =>
              getByName(name).pipe(
                Effect.flatMap((row) =>
                  row ? Effect.succeed(row) : findOwnedProcess(id, env.project, location),
                ),
              ),
            ),
          );
        current = created ?? undefined;
      }

      if (current === undefined) {
        return yield* new ProcessNotResolved({ resourceName: name });
      }

      const resolvedName = current.name ?? name;
      const desiredOrigin = defaultOrigin(news.origin ?? current.origin);
      const displayName = news.displayName ?? current.displayName;
      const originChanged = !sameJson(current.origin, desiredOrigin);
      const displayNameChanged = !sameText(current.displayName, displayName);
      const attributesChanged = !sameJson(current.attributes, attributes);

      if (originChanged || displayNameChanged || attributesChanged) {
        current = yield* datalineage.patchProjectsLocationsProcesses({
          name: resolvedName,
          body: {
            name: resolvedName,
            displayName,
            origin: desiredOrigin,
            attributes,
          },
        });
      }

      return toAttrs(current, env.project, env.region);
    }),

    delete: Effect.fn(function* ({ output }) {
      if (!output.name) return;
      const operation = yield* datalineage
        .deleteProjectsLocationsProcesses({
          name: output.name,
          allowMissing: true,
        })
        .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
      if (operation !== undefined) {
        yield* waitForOperation(operation, { notFoundOk: true });
      }
      yield* waitUntilGone(getByName(output.name), output.name);
    }),
  });