sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ DataBrew/ Recipe.ts

import * as databrew from "@distilled.cloud/aws/databrew";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, hasAlchemyTags } from "alchemy/Tags";
import * as Effect from "effect/Effect";
import * as Stream from "effect/Stream";
import { AWSEnvironment } from "../Environment.ts";
import type { Providers } from "../Providers.ts";
import {
  canonicalJson,
  cleanMap,
  databrewArn,
  fetchObservedTags,
  retryWhileConflict,
  syncTags,
} from "./internal.ts";

/** A single transformation step in a DataBrew recipe. */
export interface RecipeStep {
  /** The transformation to apply. */
  action: {
    /**
     * The DataBrew operation name, e.g. `UPPER_CASE`, `REMOVE_VALUES`,
     * `RENAME`. See the DataBrew recipe-action reference for the full list.
     */
    operation: string;
    /** Operation parameters, e.g. `{ sourceColumn: "name" }`. */
    parameters?: Record<string, string>;
  };
  /** Conditions that must hold for the step to apply to a row. */
  conditionExpressions?: {
    /** The condition name, e.g. `LESS_THAN`, `IS_MISSING`. */
    condition: string;
    /** The value to compare against (JSON-encoded for lists). */
    value?: string;
    /** The column the condition applies to. */
    targetColumn: string;
  }[];
}

export interface RecipeProps {
  /**
   * Name of the recipe. If omitted, a unique name is generated. Changing
   * the name replaces the recipe.
   * @default a generated physical name
   */
  recipeName?: string;
  /**
   * A description of the recipe.
   */
  description?: string;
  /**
   * The ordered transformation steps applied to the data.
   */
  steps: RecipeStep[];
  /**
   * Publish a new numbered version whenever the working steps change (and
   * once on create). Recipe jobs consume the latest *published* version by
   * default, so set this when the recipe feeds a `DataBrew.Job`.
   * @default false
   */
  publish?: boolean;
  /**
   * Tags to apply to the recipe. Merged with internal Alchemy tags.
   */
  tags?: Record<string, string>;
}

export interface Recipe extends Resource<
  "AWS.DataBrew.Recipe",
  RecipeProps,
  {
    /** Name of the recipe. */
    recipeName: string;
    /** ARN of the recipe. */
    recipeArn: string;
    /** Latest published version (e.g. `"1.0"`), or `"LATEST_WORKING"` if never published. */
    recipeVersion: string;
  },
  {},
  Providers
> {}

/**
 * An AWS Glue DataBrew recipe — an ordered list of data-transformation steps
 * (rename, filter, case conversion, etc.). Edits modify the `LATEST_WORKING`
 * version; setting `publish: true` snapshots numbered versions that recipe
 * jobs consume.
 * ### Creating Recipes
 * **Example:** Simple Transform Recipe
 * ```typescript
 * import * as AWS from "@alchemy.run/aws";
 *
 * const recipe = yield* AWS.DataBrew.Recipe("Clean", {
 *   description: "normalize customer names",
 *   steps: [
 *     {
 *       action: {
 *         operation: "UPPER_CASE",
 *         parameters: { sourceColumn: "name" },
 *       },
 *     },
 *   ],
 * });
 * ```
 *
 * **Example:** Published Recipe for Jobs
 * ```typescript
 * // publish: true snapshots a numbered version (1.0, 2.0, ...) whenever the
 * // steps change — recipe jobs run the latest published version by default.
 * const recipe = yield* AWS.DataBrew.Recipe("Clean", {
 *   publish: true,
 *   steps: [
 *     {
 *       action: {
 *         operation: "REMOVE_VALUES",
 *         parameters: { sourceColumn: "email" },
 *       },
 *       conditionExpressions: [
 *         { condition: "IS_MISSING", targetColumn: "email" },
 *       ],
 *     },
 *   ],
 * });
 * ```
 *
 * @resource
 */
export const Recipe = Resource<Recipe>("AWS.DataBrew.Recipe");

export const buildSteps = (steps: RecipeStep[]) =>
  steps.map((step) => ({
    Action: {
      Operation: step.action.operation,
      Parameters: step.action.parameters,
    },
    ConditionExpressions: step.conditionExpressions?.map((expr) => ({
      Condition: expr.condition,
      Value: expr.value,
      TargetColumn: expr.targetColumn,
    })),
  }));

/** Numeric-aware max over published version strings like `"2.0"`. */
const latestPublished = (versions: (string | undefined)[]) => {
  const numeric = versions
    .filter((v): v is string => v !== undefined && !v.startsWith("LATEST"))
    .sort((a, b) => Number.parseFloat(a) - Number.parseFloat(b));
  return numeric.at(-1);
};

export const RecipeProvider = () =>
  Provider.effect(
    Recipe,
    Effect.gen(function* () {
      const createName = Effect.fn(function* (
        id: string,
        props: { recipeName?: string | undefined },
      ) {
        return props.recipeName ?? (yield* createPhysicalName({ id, maxLength: 255 }));
      });

      const observeWorking = Effect.fn(function* (name: string) {
        return yield* databrew
          .describeRecipe({ Name: name, RecipeVersion: "LATEST_WORKING" })
          .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
      });

      const publishedVersions = Effect.fn(function* (name: string) {
        const pages = yield* databrew.listRecipeVersions
          .pages({ Name: name })
          .pipe(Stream.runCollect);
        return Array.from(pages)
          .flatMap((page) => page.Recipes ?? [])
          .map((r) => r.RecipeVersion);
      });

      return Recipe.Provider.of({
        stables: ["recipeName", "recipeArn"],

        list: () =>
          Effect.gen(function* () {
            const { accountId, region } = yield* AWSEnvironment.current;
            // LATEST_WORKING enumerates every recipe (the default listing
            // only returns recipes with a published version).
            const workingPages = yield* databrew.listRecipes
              .pages({ RecipeVersion: "LATEST_WORKING" })
              .pipe(Stream.runCollect);
            const publishedPages = yield* databrew.listRecipes.pages({}).pipe(Stream.runCollect);
            const published = new Map(
              Array.from(publishedPages)
                .flatMap((page) => page.Recipes ?? [])
                .map((r) => [r.Name, r.RecipeVersion]),
            );
            return Array.from(workingPages)
              .flatMap((page) => page.Recipes ?? [])
              .map((r) => ({
                recipeName: r.Name,
                recipeArn: r.ResourceArn ?? databrewArn(region, accountId, "recipe", r.Name),
                recipeVersion: published.get(r.Name) ?? "LATEST_WORKING",
              }));
          }),

        read: Effect.fn(function* ({ id, olds, output }) {
          const { accountId, region } = yield* AWSEnvironment.current;
          const name = output?.recipeName ?? (yield* createName(id, olds ?? {}));
          const recipe = yield* observeWorking(name);
          if (recipe === undefined) return undefined;
          const arn = recipe.ResourceArn ?? databrewArn(region, accountId, "recipe", name);
          const versions = yield* publishedVersions(name);
          const attrs = {
            recipeName: name,
            recipeArn: arn,
            recipeVersion: latestPublished(versions) ?? "LATEST_WORKING",
          };
          const tags = cleanMap(recipe.Tags);
          return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
        }),

        diff: Effect.fn(function* ({ id, news, olds }) {
          if (!isResolved(news)) return undefined;
          const oldName = yield* createName(id, olds);
          const newName = yield* createName(id, news);
          if (oldName !== newName) return { action: "replace" } as const;
          // steps/description are UpdateRecipe-able; publish is a sync step
        }),

        reconcile: Effect.fn(function* ({ id, news, output, session }) {
          const { accountId, region } = yield* AWSEnvironment.current;
          const name = output?.recipeName ?? (yield* createName(id, news));
          const internalTags = yield* createInternalTags(id);
          const desiredTags = { ...news.tags, ...internalTags };
          const desiredSteps = buildSteps(news.steps);

          // 1. OBSERVE the working copy
          let recipe = yield* observeWorking(name);
          let changed = false;

          // 2. ENSURE
          if (recipe === undefined) {
            yield* databrew
              .createRecipe({
                Name: name,
                Description: news.description,
                Steps: desiredSteps,
                Tags: desiredTags,
              })
              .pipe(Effect.catchTag("ConflictException", () => Effect.void));
            changed = true;
            recipe = yield* observeWorking(name);
          } else if (
            canonicalJson(recipe.Steps ?? []) !== canonicalJson(desiredSteps) ||
            (recipe.Description ?? undefined) !== news.description
          ) {
            // 3. SYNC the working copy only when observed differs
            yield* databrew.updateRecipe({
              Name: name,
              Description: news.description,
              Steps: desiredSteps,
            });
            changed = true;
          }

          // 3b. SYNC published versions — publish when the working copy
          // changed, or when publishing is desired but nothing is published.
          let versions = yield* publishedVersions(name);
          if (news.publish === true && (changed || versions.length === 0)) {
            yield* databrew.publishRecipe({
              Name: name,
              Description: news.description,
            });
            versions = yield* publishedVersions(name);
          }

          const arn = recipe?.ResourceArn ?? databrewArn(region, accountId, "recipe", name);

          // 3c. SYNC TAGS against observed cloud tags
          const observedTags = yield* fetchObservedTags(arn);
          yield* syncTags(arn, observedTags, desiredTags);

          yield* session.note(name);
          return {
            recipeName: name,
            recipeArn: arn,
            recipeVersion: latestPublished(versions) ?? "LATEST_WORKING",
          };
        }),

        delete: Effect.fn(function* ({ output }) {
          const name = output.recipeName;
          // A recipe is deleted by deleting all of its versions: published
          // versions first (batched), then LATEST_WORKING.
          const versions = yield* databrew.listRecipeVersions.pages({ Name: name }).pipe(
            Stream.runCollect,
            Effect.map((pages) =>
              Array.from(pages)
                .flatMap((page) => page.Recipes ?? [])
                .map((r) => r.RecipeVersion)
                .filter((v): v is string => v !== undefined),
            ),
            Effect.catchTag("ResourceNotFoundException", () => Effect.succeed([] as string[])),
          );
          if (versions.length > 0) {
            yield* retryWhileConflict(
              databrew.batchDeleteRecipeVersion({
                Name: name,
                RecipeVersions: versions,
              }),
            ).pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
          }
          yield* retryWhileConflict(
            databrew.deleteRecipeVersion({
              Name: name,
              RecipeVersion: "LATEST_WORKING",
            }),
          ).pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
        }),
      });
    }),
  );