sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ Glue/ Job.ts

import * as glue from "@distilled.cloud/aws/glue";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, hasAlchemyTags } from "alchemy/Tags";
import { toWireMinutes } from "alchemy/Util/Duration";
import type * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Stream from "effect/Stream";
import { AWSEnvironment } from "../Environment.ts";
import type { Providers } from "../Providers.ts";
import {
  fetchObservedTags,
  jobArn,
  reapGlueLogStreams,
  retryWhileRoleNotAssumable,
  syncTags,
} from "./internal.ts";

/** Shared, service-managed log groups Glue writes job-run streams to. */
const GLUE_JOB_LOG_GROUPS = [
  "/aws-glue/python-jobs/output",
  "/aws-glue/python-jobs/error",
  "/aws-glue/jobs/output",
  "/aws-glue/jobs/error",
  "/aws-glue/jobs/logs-v2",
];

export interface JobCommand {
  /**
   * The command name: `glueetl` (Spark), `pythonshell` (Python shell), or
   * `gluestreaming` (Spark streaming).
   */
  name: "glueetl" | "pythonshell" | "gluestreaming" | (string & {});
  /** The S3 path to the job script, e.g. `s3://my-bucket/scripts/job.py`. */
  scriptLocation?: string;
  /** For `pythonshell`, the Python version (`3` or `3.9`). */
  pythonVersion?: string;
  /** The Ray runtime version (for Ray jobs). */
  runtime?: string;
}

export interface JobProps {
  /**
   * Name of the job. If omitted, a unique name is generated. Changing the
   * name replaces the job.
   * @default a generated physical name
   */
  jobName?: string;
  /**
   * The IAM role (ARN or name) the job assumes to run.
   */
  role: string;
  /**
   * The job's execution command (script location + runtime).
   */
  command: JobCommand;
  /**
   * A description of the job.
   */
  description?: string;
  /**
   * Default arguments passed to the script (keys are `--`-prefixed), e.g.
   * `{ "--TempDir": "s3://.../tmp/", "--job-language": "python" }`.
   */
  defaultArguments?: Record<string, string>;
  /**
   * Arguments that cannot be overridden at run time.
   */
  nonOverridableArguments?: Record<string, string>;
  /**
   * Glue connection names the job uses.
   */
  connections?: string[];
  /**
   * Max retries before the job run is considered failed.
   */
  maxRetries?: number;
  /**
   * Job timeout, e.g. `"1 hour"` or `Duration.minutes(60)`. Rounded to
   * whole minutes on the wire (the Glue API unit).
   */
  timeout?: Duration.Input;
  /**
   * Max Glue data processing units (DPUs) for `pythonshell` (0.0625 or 1).
   * Mutually exclusive with `numberOfWorkers`/`workerType`.
   */
  maxCapacity?: number;
  /**
   * The Glue version (e.g. `4.0`, `3.0`).
   */
  glueVersion?: string;
  /**
   * The number of workers of `workerType` allocated (Spark jobs).
   */
  numberOfWorkers?: number;
  /**
   * The worker type for Spark jobs: `Standard`, `G.1X`, `G.2X`, `G.025X`,
   * `G.4X`, `G.8X`, `Z.2X`.
   */
  workerType?: string;
  /**
   * Execution class: `STANDARD` or `FLEX`.
   */
  executionClass?: "STANDARD" | "FLEX";
  /**
   * Concurrency configuration.
   */
  executionProperty?: {
    /** Maximum number of concurrent runs allowed. */
    maxConcurrentRuns?: number;
  };
  /**
   * Tags to apply to the job. Merged with internal Alchemy tags.
   */
  tags?: Record<string, string>;
}

export interface Job extends Resource<
  "AWS.Glue.Job",
  JobProps,
  {
    /** The name of the job. */
    jobName: string;
    /** The ARN of the job. */
    jobArn: string;
    /** The IAM role the job assumes to run. */
    role: string;
  },
  {},
  Providers
> {}

/**
 * An AWS Glue job — a Spark (`glueetl`), Python shell (`pythonshell`), or
 * streaming ETL job definition (script in S3 + IAM role + arguments). The
 * definition lifecycle is instant and free; job *runs* are billed and are
 * started via `startJobRun`.
 * ### Creating Jobs
 * **Example:** Python Shell Job
 * ```typescript
 * import * as AWS from "@alchemy.run/aws";
 *
 * const job = yield* AWS.Glue.Job("Etl", {
 *   role: jobRole.roleArn,
 *   command: {
 *     name: "pythonshell",
 *     pythonVersion: "3.9",
 *     scriptLocation: "s3://my-bucket/scripts/etl.py",
 *   },
 *   maxCapacity: 0.0625,
 *   glueVersion: "3.0",
 *   defaultArguments: { "--job-language": "python" },
 * });
 * ```
 *
 * **Example:** Spark ETL Job
 * ```typescript
 * const job = yield* AWS.Glue.Job("SparkEtl", {
 *   role: jobRole.roleArn,
 *   command: {
 *     name: "glueetl",
 *     scriptLocation: "s3://my-bucket/scripts/spark.py",
 *   },
 *   glueVersion: "4.0",
 *   workerType: "G.1X",
 *   numberOfWorkers: 2,
 *   timeout: "1 hour",
 * });
 * ```
 *
 * ### Running Jobs
 * **Example:** Start a Job Run from a Lambda
 * ```typescript
 * // init
 * const startJobRun = yield* AWS.Glue.StartJobRun(job);
 *
 * // runtime
 * const { JobRunId } = yield* startJobRun({});
 * ```
 *
 * @resource
 */
export const Job = Resource<Job>("AWS.Glue.Job");

export const JobProvider = () =>
  Provider.effect(
    Job,
    Effect.gen(function* () {
      const createName = Effect.fn(function* (id: string, props: { jobName?: string | undefined }) {
        return props.jobName ?? (yield* createPhysicalName({ id, maxLength: 255 }));
      });

      const observe = Effect.fn(function* (name: string) {
        return yield* glue.getJob({ JobName: name }).pipe(
          Effect.map((r) => r.Job),
          Effect.catchTag("EntityNotFoundException", () => Effect.undefined),
        );
      });

      const buildCommand = (command: JobCommand) => ({
        Name: command.name,
        ScriptLocation: command.scriptLocation,
        PythonVersion: command.pythonVersion,
        Runtime: command.runtime,
      });

      const buildDefinition = (props: JobProps) => ({
        Role: props.role,
        Command: buildCommand(props.command),
        Description: props.description,
        DefaultArguments: props.defaultArguments,
        NonOverridableArguments: props.nonOverridableArguments,
        Connections:
          props.connections !== undefined ? { Connections: props.connections } : undefined,
        MaxRetries: props.maxRetries,
        Timeout: toWireMinutes(props.timeout),
        MaxCapacity: props.maxCapacity,
        GlueVersion: props.glueVersion,
        NumberOfWorkers: props.numberOfWorkers,
        WorkerType: props.workerType,
        ExecutionClass: props.executionClass,
        ExecutionProperty: props.executionProperty
          ? { MaxConcurrentRuns: props.executionProperty.maxConcurrentRuns }
          : undefined,
      });

      return Job.Provider.of({
        stables: ["jobName", "jobArn"],

        list: () =>
          Effect.gen(function* () {
            const { accountId, region } = yield* AWSEnvironment.current;
            const pages = yield* glue.getJobs.pages({}).pipe(Stream.runCollect);
            return Array.from(pages)
              .flatMap((page) => page.Jobs ?? [])
              .filter((j) => j.Name !== undefined)
              .map((j) => ({
                jobName: j.Name!,
                jobArn: jobArn(region, accountId, j.Name!),
                role: j.Role ?? "",
              }));
          }),

        read: Effect.fn(function* ({ id, olds, output }) {
          const { accountId, region } = yield* AWSEnvironment.current;
          const name = output?.jobName ?? (yield* createName(id, olds ?? {}));
          const job = yield* observe(name);
          if (job?.Name === undefined) return undefined;
          const arn = jobArn(region, accountId, job.Name);
          const attrs = {
            jobName: job.Name,
            jobArn: arn,
            role: job.Role ?? "",
          };
          const tags = yield* fetchObservedTags(arn);
          return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
        }),

        diff: Effect.fn(function* ({ id, news, olds }) {
          if (!isResolved(news)) return undefined;
          const oldName = yield* createName(id, olds);
          const newName = yield* createName(id, news);
          if (oldName !== newName) return { action: "replace" } as const;
          // everything else is UpdateJob-able
        }),

        reconcile: Effect.fn(function* ({ id, news, output, session }) {
          const { accountId, region } = yield* AWSEnvironment.current;
          const name = output?.jobName ?? (yield* createName(id, news));
          const internalTags = yield* createInternalTags(id);
          const desiredTags = { ...news.tags, ...internalTags };
          const arn = jobArn(region, accountId, name);

          // 1. OBSERVE
          let job = yield* observe(name);

          // 2. ENSURE / 3. SYNC
          if (job === undefined) {
            yield* retryWhileRoleNotAssumable(
              glue.createJob({
                Name: name,
                ...buildDefinition(news),
                Tags: desiredTags,
              }),
            ).pipe(Effect.catchTag("AlreadyExistsException", () => Effect.void));
          } else {
            // UpdateJob replaces the full JobUpdate (Name is not part of it).
            yield* retryWhileRoleNotAssumable(
              glue.updateJob({
                JobName: name,
                JobUpdate: buildDefinition(news),
              }),
            );
          }

          // 3b. SYNC TAGS
          const observedTags = yield* fetchObservedTags(arn);
          yield* syncTags(arn, observedTags, desiredTags);

          job = yield* observe(name);

          yield* session.note(name);
          return {
            jobName: name,
            jobArn: arn,
            role: job?.Role ?? news.role,
          };
        }),

        delete: Effect.fn(function* ({ output }) {
          // Glue writes each run's logs to streams prefixed by the run id in
          // SHARED, service-managed groups (`/aws-glue/python-jobs/output`,
          // `/aws-glue/jobs/logs-v2`, …). deleteJob leaves them behind and
          // drops the run history, so enumerate our runs and reap their
          // streams first (never the shared groups themselves).
          const job = yield* observe(output.jobName);
          const runIds = yield* glue.getJobRuns.items({ JobName: output.jobName }).pipe(
            Stream.map((run) => run.Id),
            Stream.filter((runId): runId is string => runId !== undefined),
            Stream.runCollect,
            Effect.map((chunk) => Array.from(chunk)),
            Effect.catchTag("EntityNotFoundException", () => Effect.succeed([] as string[])),
          );
          if (runIds.length > 0) {
            const continuousLogGroup = job?.DefaultArguments?.["--continuous-log-logGroup"];
            yield* reapGlueLogStreams(
              [
                ...GLUE_JOB_LOG_GROUPS,
                ...(continuousLogGroup === undefined ? [] : [continuousLogGroup]),
              ],
              runIds,
            );
          }

          // DeleteJob is idempotent — the API returns success (not an error)
          // when the job definition is already gone.
          yield* glue.deleteJob({ JobName: output.jobName });
        }),
      });
    }),
  );