sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ OSIS/ Pipeline.ts

import * as osis from "@distilled.cloud/aws/osis";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Stream from "effect/Stream";
import type { Providers } from "../Providers.ts";
import {
  jsonEquals,
  readPipelineTags,
  retryWhilePipelineConflict,
  waitForPipelineSettled,
} from "./internal.ts";

export class OSISPipelineUnexpectedResponse extends Data.TaggedError(
  "OSISPipelineUnexpectedResponse",
)<{
  readonly message: string;
  readonly cause?: unknown;
}> {}

export interface PipelineLogPublishingOptions {
  /**
   * Whether pipeline logs are published to CloudWatch Logs.
   */
  isLoggingEnabled?: boolean;
  /**
   * CloudWatch Logs destination. The log group name must start with
   * `/aws/vendedlogs/`.
   */
  cloudWatchLogDestination?: {
    /** Name of the CloudWatch Logs log group, e.g. `/aws/vendedlogs/OpenSearchIngestion/my-pipeline`. */
    logGroup: string;
  };
}

export interface PipelineBufferOptions {
  /**
   * Whether persistent buffering is enabled for the pipeline.
   */
  persistentBufferEnabled: boolean;
}

export interface PipelineEncryptionAtRestOptions {
  /**
   * ARN of the customer-managed KMS key used to encrypt buffer data.
   */
  kmsKeyArn: string;
}

export interface PipelineVpcOptions {
  /**
   * Subnet IDs the pipeline's VPC endpoint is placed into. Changing VPC
   * options replaces the pipeline.
   */
  subnetIds: string[];
  /**
   * Security group IDs applied to the pipeline's VPC endpoint.
   */
  securityGroupIds?: string[];
}

export interface PipelineProps {
  /**
   * Name of the pipeline. 3-28 characters; lowercase letters, numbers, and
   * hyphens. If omitted, a deterministic physical name is generated.
   * Changing the name replaces the pipeline.
   */
  pipelineName?: string;
  /**
   * Minimum number of Ingestion OpenSearch Compute Units (OCUs) the pipeline
   * scales down to. Minimum 1.
   */
  minUnits: number;
  /**
   * Maximum number of Ingestion OpenSearch Compute Units (OCUs) the pipeline
   * scales up to.
   */
  maxUnits: number;
  /**
   * Data Prepper pipeline configuration in YAML (must start with
   * `version: "2"`). Blueprints for common source/sink topologies are
   * available via the `listPipelineBlueprints`/`getPipelineBlueprint` APIs.
   */
  pipelineConfigurationBody: string;
  /**
   * CloudWatch Logs publishing configuration for pipeline logs.
   */
  logPublishingOptions?: PipelineLogPublishingOptions;
  /**
   * Persistent buffering for the pipeline's ingest data.
   */
  bufferOptions?: PipelineBufferOptions;
  /**
   * Customer-managed KMS encryption for buffer data.
   * @default an AWS-owned key
   */
  encryptionAtRestOptions?: PipelineEncryptionAtRestOptions;
  /**
   * VPC placement for the pipeline's ingest endpoint. Omit for a public
   * endpoint. Changing VPC options replaces the pipeline.
   */
  vpcOptions?: PipelineVpcOptions;
  /**
   * IAM role the pipeline assumes to write to its sinks (overrides the
   * `sts_role_arn` inside the configuration body where supported).
   */
  pipelineRoleArn?: string;
  /**
   * User-defined tags for the pipeline.
   */
  tags?: Record<string, string>;
}

export interface Pipeline extends Resource<
  "AWS.OSIS.Pipeline",
  PipelineProps,
  {
    /**
     * Name of the pipeline.
     */
    pipelineName: string;
    /**
     * ARN of the pipeline.
     */
    pipelineArn: string;
    /**
     * Pipeline status (e.g. `ACTIVE`, `CREATING`, `UPDATING`).
     */
    status: string;
    /**
     * Minimum Ingestion OCUs the pipeline scales down to.
     */
    minUnits: number | undefined;
    /**
     * Maximum Ingestion OCUs the pipeline scales up to.
     */
    maxUnits: number | undefined;
    /**
     * URLs to ingest data into the pipeline.
     */
    ingestEndpointUrls: string[] | undefined;
    /**
     * Tags on the pipeline.
     */
    tags: Record<string, string>;
  },
  never,
  Providers
> {}

/**
 * An Amazon OpenSearch Ingestion (OSIS) pipeline — a managed Data Prepper
 * pipeline that ingests, transforms, and delivers data to OpenSearch domains,
 * serverless collections, or S3.
 *
 * Pipelines take roughly 5-10 minutes to provision and are billed per
 * Ingestion-OCU-hour while they exist (minimum 1 OCU). Destroy pipelines you
 * are not using.
 * ### Creating a Pipeline
 * **Example:** HTTP Source to S3 Sink
 * ```typescript
 * const pipeline = yield* Pipeline("Logs", {
 *   minUnits: 1,
 *   maxUnits: 1,
 *   pipelineConfigurationBody: Output.interpolate`version: "2"
 * log-pipeline:
 *   source:
 *     http:
 *       path: "/logs"
 *   sink:
 *     - s3:
 *         aws:
 *           sts_role_arn: "${role.roleArn}"
 *           region: "us-west-2"
 *         bucket: "${bucket.bucketName}"
 *         threshold:
 *           event_collect_timeout: "60s"
 *         codec:
 *           ndjson:
 * `,
 * });
 * ```
 *
 * **Example:** Pipeline with CloudWatch Logging
 * ```typescript
 * const pipeline = yield* Pipeline("Logs", {
 *   minUnits: 1,
 *   maxUnits: 2,
 *   pipelineConfigurationBody: configYaml,
 *   logPublishingOptions: {
 *     isLoggingEnabled: true,
 *     cloudWatchLogDestination: {
 *       logGroup: "/aws/vendedlogs/OpenSearchIngestion/logs",
 *     },
 *   },
 * });
 * ```
 *
 * @resource
 */
export const Pipeline = Resource<Pipeline>("AWS.OSIS.Pipeline");

const toLogPublishingOptions = (
  options: PipelineLogPublishingOptions | undefined,
): osis.LogPublishingOptions | undefined =>
  options === undefined
    ? undefined
    : {
        IsLoggingEnabled: options.isLoggingEnabled,
        CloudWatchLogDestination:
          options.cloudWatchLogDestination !== undefined
            ? { LogGroup: options.cloudWatchLogDestination.logGroup }
            : undefined,
      };

export const PipelineProvider = () =>
  Provider.effect(
    Pipeline,
    Effect.gen(function* () {
      const toName = (id: string, props: Partial<PipelineProps>) =>
        props.pipelineName
          ? Effect.succeed(props.pipelineName)
          : createPhysicalName({ id, maxLength: 28, lowercase: true });

      const readPipeline = Effect.fn(function* (name: string) {
        return yield* osis.getPipeline({ PipelineName: name }).pipe(
          Effect.map((response) => response.Pipeline),
          Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
        );
      });

      const waitForActive = Effect.fn(function* (name: string) {
        const pipeline = yield* waitForPipelineSettled(name, readPipeline(name));
        if (pipeline === undefined) {
          return yield* new OSISPipelineUnexpectedResponse({
            message: `OSIS pipeline '${name}' not found while waiting`,
          });
        }
        return pipeline;
      });

      const toAttrs = Effect.fn(function* (pipeline: osis.Pipeline) {
        if (!pipeline.PipelineName || !pipeline.PipelineArn) {
          return yield* new OSISPipelineUnexpectedResponse({
            message: `OSIS pipeline '${pipeline.PipelineName}' is missing its ARN`,
          });
        }
        return {
          pipelineName: pipeline.PipelineName,
          pipelineArn: pipeline.PipelineArn,
          status: pipeline.Status ?? "ACTIVE",
          minUnits: pipeline.MinUnits,
          maxUnits: pipeline.MaxUnits,
          ingestEndpointUrls: pipeline.IngestEndpointUrls,
          tags: yield* readPipelineTags(pipeline.PipelineArn),
        };
      });

      return {
        stables: ["pipelineName", "pipelineArn"],

        diff: Effect.fn(function* ({ id, olds, news }) {
          if (!isResolved(news)) return undefined;
          const o: Partial<PipelineProps> = olds ?? {};
          const n: Partial<PipelineProps> = news ?? {};
          if ((yield* toName(id, o)) !== (yield* toName(id, n))) {
            return { action: "replace" } as const;
          }
          // VPC options are create-only.
          if (!jsonEquals(o.vpcOptions, n.vpcOptions)) {
            return { action: "replace" } as const;
          }
        }),

        read: Effect.fn(function* ({ id, olds, output }) {
          const name = output?.pipelineName ?? (yield* toName(id, olds ?? {}));
          const pipeline = yield* readPipeline(name);
          if (pipeline === undefined) return undefined;
          const attrs = yield* toAttrs(pipeline);
          return (yield* hasAlchemyTags(id, attrs.tags)) ? attrs : Unowned(attrs);
        }),

        reconcile: Effect.fn(function* ({ id, news, output, session }) {
          const props = news!;
          const name = output?.pipelineName ?? (yield* toName(id, props));
          const internalTags = yield* createInternalTags(id);
          const desiredTags = { ...internalTags, ...props.tags };

          // 1. Observe — cloud state is authoritative.
          let observed = yield* readPipeline(name);

          // 2. Ensure — create if missing; tolerate AlreadyExists as a race.
          if (observed === undefined) {
            yield* osis
              .createPipeline({
                PipelineName: name,
                MinUnits: props.minUnits,
                MaxUnits: props.maxUnits,
                PipelineConfigurationBody: props.pipelineConfigurationBody,
                LogPublishingOptions: toLogPublishingOptions(props.logPublishingOptions),
                BufferOptions:
                  props.bufferOptions !== undefined
                    ? {
                        PersistentBufferEnabled: props.bufferOptions.persistentBufferEnabled,
                      }
                    : undefined,
                EncryptionAtRestOptions:
                  props.encryptionAtRestOptions !== undefined
                    ? { KmsKeyArn: props.encryptionAtRestOptions.kmsKeyArn }
                    : undefined,
                VpcOptions:
                  props.vpcOptions !== undefined
                    ? {
                        SubnetIds: props.vpcOptions.subnetIds,
                        SecurityGroupIds: props.vpcOptions.securityGroupIds,
                      }
                    : undefined,
                PipelineRoleArn: props.pipelineRoleArn,
                Tags: Object.entries(desiredTags).map(([Key, Value]) => ({
                  Key,
                  Value,
                })),
              })
              .pipe(Effect.catchTag("ResourceAlreadyExistsException", () => Effect.void));
          }

          // Creation and in-flight updates both surface as transitional
          // statuses; wait (bounded) so update calls do not conflict.
          observed = yield* waitForActive(name);

          // 3. Sync — compute the update delta from OBSERVED state.
          const update: osis.UpdatePipelineRequest = { PipelineName: name };
          let mutated = false;
          if (props.minUnits !== observed.MinUnits) {
            update.MinUnits = props.minUnits;
            mutated = true;
          }
          if (props.maxUnits !== observed.MaxUnits) {
            update.MaxUnits = props.maxUnits;
            mutated = true;
          }
          if (
            props.pipelineConfigurationBody.trim() !==
            (observed.PipelineConfigurationBody ?? "").trim()
          ) {
            update.PipelineConfigurationBody = props.pipelineConfigurationBody;
            mutated = true;
          }
          const desiredLogging = toLogPublishingOptions(props.logPublishingOptions);
          if (
            desiredLogging !== undefined &&
            !jsonEquals(
              {
                IsLoggingEnabled: observed.LogPublishingOptions?.IsLoggingEnabled,
                CloudWatchLogDestination: observed.LogPublishingOptions?.CloudWatchLogDestination,
              },
              desiredLogging,
            )
          ) {
            update.LogPublishingOptions = desiredLogging;
            mutated = true;
          }
          if (
            props.bufferOptions !== undefined &&
            props.bufferOptions.persistentBufferEnabled !==
              observed.BufferOptions?.PersistentBufferEnabled
          ) {
            update.BufferOptions = {
              PersistentBufferEnabled: props.bufferOptions.persistentBufferEnabled,
            };
            mutated = true;
          }
          if (
            props.encryptionAtRestOptions !== undefined &&
            props.encryptionAtRestOptions.kmsKeyArn !== observed.EncryptionAtRestOptions?.KmsKeyArn
          ) {
            update.EncryptionAtRestOptions = {
              KmsKeyArn: props.encryptionAtRestOptions.kmsKeyArn,
            };
            mutated = true;
          }
          if (
            props.pipelineRoleArn !== undefined &&
            props.pipelineRoleArn !== observed.PipelineRoleArn
          ) {
            update.PipelineRoleArn = props.pipelineRoleArn;
            mutated = true;
          }

          if (mutated) {
            yield* osis.updatePipeline(update);
            observed = yield* waitForActive(name);
          }

          // 3b. Sync tags — diff against OBSERVED cloud tags.
          const arn = observed.PipelineArn;
          if (arn) {
            const observedTags = yield* readPipelineTags(arn);
            const { removed, upsert } = diffTags(observedTags, desiredTags);
            if (upsert.length > 0) {
              yield* osis.tagResource({ Arn: arn, Tags: upsert });
            }
            if (removed.length > 0) {
              yield* osis.untagResource({ Arn: arn, TagKeys: removed });
            }
          }

          yield* session.note(name);
          return yield* toAttrs(observed);
        }),

        delete: Effect.fn(function* ({ output }) {
          const name = output.pipelineName;
          // A pipeline mid-create/update rejects deletion with
          // ConflictException — wait (bounded, tolerant) for it to settle,
          // then retry conflicts through any residual transition.
          yield* waitForPipelineSettled(name, readPipeline(name)).pipe(
            Effect.orElseSucceed(() => undefined),
          );
          yield* retryWhilePipelineConflict(
            osis
              .deletePipeline({ PipelineName: name })
              .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void)),
          );
        }),

        list: () =>
          osis.listPipelines.pages({}).pipe(
            Stream.runCollect,
            Effect.map((chunk) =>
              Array.from(chunk).flatMap((page) =>
                (page.Pipelines ?? [])
                  .map((summary) => summary.PipelineName)
                  .filter((name): name is string => name !== undefined),
              ),
            ),
            Effect.flatMap(
              Effect.forEach(
                (name) =>
                  osis.getPipeline({ PipelineName: name }).pipe(
                    Effect.map((response) => response.Pipeline),
                    Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
                  ),
                { concurrency: 4 },
              ),
            ),
            Effect.map((pipelines) =>
              pipelines.filter((pipeline): pipeline is osis.Pipeline => pipeline !== undefined),
            ),
            Effect.flatMap(
              Effect.forEach((pipeline) => toAttrs(pipeline), {
                concurrency: 4,
              }),
            ),
          ),
      };
    }),
  );