sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ Dataplex/ LakesTask.ts

import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { tagRecord } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import { GcpEnvironment } from "../Environment.ts";
import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
import type { Providers } from "../Providers.ts";
import { waitForOperation } from "./operations.ts";
import {
  DataplexNotResolved,
  expandParent,
  fingerprint,
  hasAlchemyLabelMap,
  isPendingState,
  listChildResources,
  listLakes,
  listTasks,
  normalizeLocation,
  parseResourceName,
  replaceIfChanged,
  toPhysicalRfc1035,
  userLabels,
} from "./shared.ts";

const DEFAULT_TRIGGER = "ON_DEMAND";

export type TaskTriggerSpec = {
  /**
   * Trigger type. Immutable — changing it replaces the task.
   * @default "ON_DEMAND"
   */
  type?: dataplex.GoogleCloudDataplexV1TaskTriggerSpecTypeEnum | (string & {});
  /** First run after this RFC3339 timestamp. */
  startTime?: string;
  /** Temporarily disable RECURRING execution. */
  disabled?: boolean;
  /** Retry attempts before aborting. */
  maxRetries?: number;
  /** Cron schedule. Required for `RECURRING`. */
  schedule?: string;
};

export type TaskExecutionSpec = {
  /** Arguments interpolated before passing to the driver. */
  args?: Record<string, string>;
  /**
   * Service account used to execute the task. If omitted, the project
   * Compute Engine default service account is used.
   */
  serviceAccount?: string;
  /** Project in which jobs run. Defaults to the lake project. */
  project?: string;
  /** Maximum job duration (e.g. `"3600s"`). */
  maxJobExecutionLifetime?: string;
  /** Cloud KMS key used to encrypt job data. */
  kmsKey?: string;
};

export type TaskInfrastructureSpec = {
  /** Dataproc Serverless executor counts. */
  batch?: {
    executorsCount?: number;
    maxExecutorsCount?: number;
  };
  /** Container image runtime. */
  containerImage?: {
    image?: string;
    javaJars?: string[];
    pythonPackages?: string[];
    properties?: Record<string, string>;
  };
  /** VPC network used to run the job. */
  vpcNetwork?: {
    network?: string;
    subNetwork?: string;
    networkTags?: string[];
  };
};

export type TaskSparkConfig = {
  /** Cloud Storage URI of the jar containing the main class. */
  mainJarFileUri?: string;
  /** Driver main class. */
  mainClass?: string;
  /** Cloud Storage URI of the main Python file. */
  pythonScriptFile?: string;
  /** Cloud Storage URI of a SQL query file. */
  sqlScriptFile?: string;
  /** Inline SQL query text. */
  sqlScript?: string;
  /** Working-directory files. */
  fileUris?: string[];
  /** Archives extracted into the working directory. */
  archiveUris?: string[];
  /** Infrastructure used to run the Spark job. */
  infrastructureSpec?: TaskInfrastructureSpec;
};

export type TaskNotebookConfig = {
  /** Path to the input notebook. */
  notebook: string;
  /** Infrastructure used to run the notebook. */
  infrastructureSpec?: TaskInfrastructureSpec;
  /** Working-directory files. */
  fileUris?: string[];
  /** Archives extracted into the working directory. */
  archiveUris?: string[];
};

export type LakesTaskProps = {
  /**
   * Parent lake. Full name
   * `projects/{project}/locations/{location}/lakes/{lake}` or the lake id
   * (combined with `location`). Immutable — changing it replaces the task.
   */
  lake: string;
  /**
   * Region used when `lake` is a bare id.
   * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
   */
  location?: string;
  /**
   * Task id. Immutable — changing it replaces the task.
   */
  taskId?: string;
  /**
   * Human-readable description.
   */
  description?: string;
  /**
   * User-friendly display name.
   */
  displayName?: string;
  /**
   * User labels. Alchemy ownership labels are merged in automatically.
   */
  labels?: Record<string, string>;
  /**
   * How often and when the task runs.
   * @default { type: "ON_DEMAND" }
   */
  triggerSpec?: TaskTriggerSpec;
  /**
   * How the task is executed (service account, args, encryption).
   */
  executionSpec?: TaskExecutionSpec;
  /**
   * Spark workload. Mutually exclusive with `notebook`.
   */
  spark?: TaskSparkConfig;
  /**
   * Notebook workload. Mutually exclusive with `spark`.
   */
  notebook?: TaskNotebookConfig;
};

export type LakesTask = Resource<
  "GCP.Dataplex.LakesTask",
  LakesTaskProps,
  {
    /** Full resource name `.../lakes/{lake}/tasks/{task}`. */
    name: string;
    /** Task id. */
    taskId: string;
    /** Parent lake resource name. */
    lake: string;
    /** Project id. */
    project: string;
    /** Location id. */
    location: string;
    /** Description. */
    description: string | undefined;
    /** User-friendly display name. */
    displayName: string | undefined;
    /** User labels (Alchemy ownership labels stripped). */
    labels: Record<string, string>;
    /** Trigger type. */
    triggerType: string | undefined;
    /** Execution service account. */
    serviceAccount: string | undefined;
    /** Lifecycle state. */
    state: string | undefined;
    /** Server-assigned uid. */
    uid: string | undefined;
    /** RFC3339 creation timestamp. */
    createTime: string | undefined;
    /** RFC3339 last-update timestamp. */
    updateTime: string | undefined;
  },
  never,
  Providers
>;

/**
 * A Dataplex lake task — a Spark or notebook job scheduled against a lake.
 *
 * Changing `lake`, `taskId`, `location`, or trigger type replaces the
 * task. Description, labels, execution spec, and Spark/notebook config
 * update in place.
 *
 * ### Creating a Task
 * **Example:** On-demand Spark SQL
 * ```typescript
 * const task = yield* GCP.Dataplex.LakesTask("SelectOne", {
 *   lake: lake.name,
 *   executionSpec: { serviceAccount },
 *   spark: { sqlScript: "SELECT 1" },
 * });
 * ```
 *
 * **Example:** Recurring notebook
 * ```typescript
 * const task = yield* GCP.Dataplex.LakesTask("Nightly", {
 *   lake: lake.name,
 *   triggerSpec: { type: "RECURRING", schedule: "0 2 * * *" },
 *   executionSpec: { serviceAccount },
 *   notebook: { notebook: "gs://bucket/job.ipynb" },
 * });
 * ```
 *
 * @resource
 * @category Dataplex
 */
export const LakesTask = Resource<LakesTask>("GCP.Dataplex.LakesTask");

export class LakesTaskNotResolved extends Data.TaggedError("GCP.Dataplex.LakesTaskNotResolved")<{
  resourceName: string;
}> {}

export class LakesTaskStillExists extends Data.TaggedError("GCP.Dataplex.LakesTaskStillExists")<{
  resourceName: string;
}> {}

const lakeOf = (lake: string, project: string, location: string) =>
  expandParent(lake, project, location, "lakes");

const resourceName = (lake: string, taskId: string) => `${lake}/tasks/${taskId}`;

const triggerBody = (
  spec: TaskTriggerSpec | undefined,
): dataplex.GoogleCloudDataplexV1TaskTriggerSpec => ({
  type: (spec?.type ?? DEFAULT_TRIGGER).toUpperCase(),
  startTime: spec?.startTime,
  disabled: spec?.disabled,
  maxRetries: spec?.maxRetries,
  schedule: spec?.schedule,
});

const executionBody = (
  spec: TaskExecutionSpec | undefined,
): dataplex.GoogleCloudDataplexV1TaskExecutionSpec => ({
  args: spec?.args,
  serviceAccount: spec?.serviceAccount,
  project: spec?.project,
  maxJobExecutionLifetime: spec?.maxJobExecutionLifetime,
  kmsKey: spec?.kmsKey,
});

const toAttrs = (task: dataplex.GoogleCloudDataplexV1Task, project: string) => {
  const name = task.name ?? "";
  const parsed = parseResourceName(name, "tasks");
  return {
    name,
    taskId: parsed.id,
    lake: parsed.parent,
    project: parsed.project || project,
    location: parsed.location,
    description: task.description,
    displayName: task.displayName,
    labels: userLabels(task.labels),
    triggerType: task.triggerSpec?.type,
    serviceAccount: task.executionSpec?.serviceAccount,
    state: task.state,
    uid: task.uid,
    createTime: task.createTime,
    updateTime: task.updateTime,
  };
};

const getByName = (name: string) =>
  name.length === 0 || name.includes("//")
    ? Effect.undefined
    : dataplex
        .getProjectsLocationsLakesTasks({ name })
        .pipe(Effect.catchTag("NotFound", () => Effect.undefined));

const waitUntilReady = (name: string) =>
  getByName(name).pipe(
    Effect.filterOrFail(
      (task): task is dataplex.GoogleCloudDataplexV1Task => task !== undefined,
      () => new LakesTaskNotResolved({ resourceName: name }),
    ),
    Effect.filterOrFail(
      (task) => !isPendingState(task.state),
      () => new DataplexNotResolved({ resourceName: name }),
    ),
    Effect.retry({
      while: (error) =>
        error._tag === "GCP.Dataplex.LakesTaskNotResolved" ||
        error._tag === "GCP.Dataplex.NotResolved" ||
        error._tag === "TooManyRequests",
      times: 10,
      schedule: Schedule.spaced("10 seconds"),
    }),
  );

const waitUntilGone = (name: string) =>
  getByName(name).pipe(
    Effect.flatMap((task) =>
      task === undefined
        ? Effect.void
        : Effect.fail(new LakesTaskStillExists({ resourceName: name })),
    ),
    Effect.retry({
      while: (error) => error._tag === "GCP.Dataplex.LakesTaskStillExists",
      times: 10,
      schedule: Schedule.spaced("3 seconds"),
    }),
  );

export const LakesTaskProvider = () =>
  Provider.succeed(LakesTask, {
    stables: ["name", "taskId", "lake", "project", "location", "uid", "createTime"],

    diff: Effect.fn(function* ({ news, olds, output }) {
      if (!isResolved(news)) return undefined;
      const env = yield* GcpEnvironment.current;
      const previousId = olds?.taskId ?? output?.taskId;
      const nextId = news.taskId ?? previousId;
      const previousLake = olds?.lake ?? output?.lake;
      const nextLake = news.lake ?? previousLake;
      const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
      const nextLocation = normalizeLocation(
        news.location ?? olds?.location ?? output?.location,
        env.region,
      );
      const previousTrigger = (
        olds?.triggerSpec?.type ??
        output?.triggerType ??
        DEFAULT_TRIGGER
      ).toUpperCase();
      const nextTrigger = (news.triggerSpec?.type ?? previousTrigger).toUpperCase();
      if (
        replaceIfChanged(previousId, nextId) ||
        replaceIfChanged(previousLake, nextLake) ||
        (output !== undefined && previousLocation !== nextLocation) ||
        previousTrigger !== nextTrigger
      ) {
        return {
          action: "replace" as const,
          deleteFirst:
            previousLake === nextLake &&
            previousLocation === nextLocation &&
            previousId !== undefined &&
            nextId === previousId,
        };
      }
      return undefined;
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(olds?.location ?? output?.location, env.region);
      const lake = lakeOf(olds?.lake ?? output?.lake ?? "", env.project, location);
      const taskId = yield* toPhysicalRfc1035(id, olds?.taskId, output?.taskId);
      const name = output?.name ?? resourceName(lake, taskId);
      const existing = yield* getByName(name);
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing, env.project);
      return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        const lakes = yield* listLakes(env.project, env.region);
        const tasks = yield* listChildResources(lakes, listTasks);
        return tasks
          .filter((task) => hasAlchemyLabelMap(task.labels))
          .map((task) => toAttrs(task, env.project));
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(news.location ?? output?.location, env.region);
      const lake = lakeOf(news.lake, env.project, location);
      const taskId = yield* toPhysicalRfc1035(id, news.taskId, output?.taskId);
      const name = output?.name ?? resourceName(lake, taskId);
      const desiredLabels = {
        ...toLabels(news.labels),
        ...(yield* createInternalLabels(id)),
      };
      const triggerSpec = triggerBody(news.triggerSpec);
      const executionSpec = executionBody(news.executionSpec);

      let current = yield* getByName(name);

      if (current === undefined) {
        const created = yield* dataplex
          .createProjectsLocationsLakesTasks({
            parent: lake,
            taskId,
            body: {
              description: news.description,
              displayName: news.displayName,
              labels: desiredLabels,
              triggerSpec,
              executionSpec,
              spark: news.spark,
              notebook: news.notebook,
            },
          })
          .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
        if (created !== undefined) {
          yield* waitForOperation(created);
        }
        current = yield* waitUntilReady(name);
      }

      if (current === undefined) {
        return yield* new LakesTaskNotResolved({ resourceName: name });
      }

      const observedLabels = tagRecord(current.labels);
      const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
      const labelsChanged = upsert.length > 0 || removed.length > 0;
      const descriptionChanged = (current.description ?? "") !== (news.description ?? "");
      const displayNameChanged = (current.displayName ?? "") !== (news.displayName ?? "");
      const triggerChanged = fingerprint(current.triggerSpec) !== fingerprint(triggerSpec);
      const executionChanged = fingerprint(current.executionSpec) !== fingerprint(executionSpec);
      const sparkChanged = fingerprint(current.spark) !== fingerprint(news.spark);
      const notebookChanged = fingerprint(current.notebook) !== fingerprint(news.notebook);

      if (
        labelsChanged ||
        descriptionChanged ||
        displayNameChanged ||
        triggerChanged ||
        executionChanged ||
        sparkChanged ||
        notebookChanged
      ) {
        const updateMask = [
          labelsChanged ? "labels" : undefined,
          descriptionChanged ? "description" : undefined,
          displayNameChanged ? "display_name" : undefined,
          triggerChanged ? "trigger_spec" : undefined,
          executionChanged ? "execution_spec" : undefined,
          sparkChanged ? "spark" : undefined,
          notebookChanged ? "notebook" : undefined,
        ].filter((field): field is string => field !== undefined);
        const operation = yield* dataplex.patchProjectsLocationsLakesTasks({
          name: current.name ?? name,
          updateMask: updateMask.join(","),
          body: {
            description: news.description,
            displayName: news.displayName,
            labels: desiredLabels,
            triggerSpec,
            executionSpec,
            spark: news.spark,
            notebook: news.notebook,
          },
        });
        yield* waitForOperation(operation);
        current = yield* waitUntilReady(current.name ?? name);
      }

      return toAttrs(current, env.project);
    }),

    delete: Effect.fn(function* ({ output }) {
      if (!output.name || output.name.includes("//")) return;
      const operation = yield* dataplex
        .deleteProjectsLocationsLakesTasks({ name: output.name })
        .pipe(
          Effect.retry({
            while: (error) => error._tag === "Conflict",
            times: 8,
            schedule: Schedule.spaced("2 seconds"),
          }),
          Effect.catchTags({
            NotFound: () => Effect.undefined,
            BadRequest: () => Effect.undefined,
          }),
        );
      if (operation !== undefined) {
        yield* waitForOperation(operation, { notFoundOk: true });
      }
      yield* waitUntilGone(output.name);
    }),
  });