sailfork / t3code-github / .repos/ alchemy-effect/ packages/ cloudflare/ src/ Pipelines/ Pipeline.ts

import * as pipelines from "@distilled.cloud/cloudflare/pipelines";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Predicate from "effect/Predicate";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
import type { Providers } from "../Providers.ts";

const TypeId = "Cloudflare.Pipelines.Pipeline" as const;
type TypeId = typeof TypeId;

export interface PipelineProps {
  /**
   * Name of the pipeline. Unique per account; must be alphanumeric and
   * underscores only. If omitted, a unique name is generated from the
   * app, stage, and logical ID.
   *
   * Pipelines have no update API, so changing this (or the `sql`)
   * triggers a replacement.
   * @default ${app}_${id}_${stage}_${suffix}
   */
  name?: string;
  /**
   * SQL statement describing the processing flow, e.g.
   * `INSERT INTO my_sink SELECT * FROM my_stream`. Streams and sinks are
   * referenced by name — interpolate `stream.name` / `sink.name` outputs
   * so the engine orders the pipeline after them on deploy (and before
   * them on destroy).
   *
   * Immutable — changing the SQL triggers a replacement. Whitespace-only
   * changes are a no-op. A changed, fully-resolved statement is validated
   * during plan, so invalid SQL fails before anything is replaced.
   */
  sql: string;
}

/**
 * Raised during plan when a pipeline's SQL is rejected by Cloudflare's
 * SQL validator (`InvalidSql`, code 1014). Nothing has been replaced or
 * deleted when this fails.
 */
export class PipelineSqlInvalid extends Data.TaggedError(
  "Cloudflare.Pipelines.PipelineSqlInvalid",
)<{
  message: string;
  sql: string;
}> {}

export interface PipelineAttributes {
  /** Cloudflare-assigned pipeline identifier. */
  pipelineId: string;
  /** Account that owns the pipeline. */
  accountId: string;
  /** Pipeline name (unique per account). */
  name: string;
  /** SQL statement of the processing flow. */
  sql: string;
  /** Current status of the pipeline. */
  status: string;
  /** When the pipeline was created. */
  createdAt: string;
  /** When the pipeline was last modified. */
  modifiedAt: string;
}

export type Pipeline = Resource<TypeId, PipelineProps, PipelineAttributes, never, Providers>;

/**
 * A Cloudflare SQL Pipeline — the transform of the Pipelines product. A
 * pipeline is a single SQL statement that reads events from a
 * {@link Stream} and writes them to a {@link Sink}, both
 * referenced by name.
 *
 * Also exported as `Cloudflare.Basin.Pipeline`.
 *
 * The SQL is fixed at creation: changing it (or the name) triggers a
 * replacement. Nothing references a pipeline downstream, so replacements
 * are cheap. A changed statement is checked with Cloudflare's SQL
 * validator during plan, so a syntax error fails the plan (with
 * {@link PipelineSqlInvalid}) instead of deleting the running pipeline.
 * References to streams/sinks that do not exist yet are not an error at
 * plan time — they are created in the same deploy.
 *
 * Pipelines is ingest-only: to process data after the sink writes it,
 * subscribe to the sink's bucket with `Cloudflare.R2.BucketEventNotification`.
 * ### Creating a Pipeline
 * **Example:** Stream → Sink passthrough
 * ```typescript
 * const stream = yield* Cloudflare.Basin.Stream("events", {});
 * const sink = yield* Cloudflare.Basin.Sink("events-sink", {
 *   type: "r2",
 *   config: { bucket: bucket.bucketName, credentials },
 * });
 *
 * const pipeline = yield* Cloudflare.Basin.Pipeline("etl", {
 *   sql: Output.interpolate`INSERT INTO ${sink.name} SELECT * FROM ${stream.name}`,
 * });
 * ```
 *
 * **Example:** Filtering transform
 * ```typescript
 * const pipeline = yield* Cloudflare.Basin.Pipeline("errors-only", {
 *   sql: Output.interpolate`INSERT INTO ${sink.name} SELECT * FROM ${stream.name} WHERE level = 'error'`,
 * });
 * ```
 *
 * @see https://developers.cloudflare.com/pipelines/
 *
 * @resource
 * @product Pipelines
 * @category Storage & Databases
 */
export const Pipeline = Resource<Pipeline>(TypeId);

/**
 * Returns true if the given value is a Pipeline resource.
 */
export const isPipeline = (value: unknown): value is Pipeline =>
  Predicate.hasProperty(value, "Type") && value.Type === TypeId;

export const PipelineProvider = () =>
  Provider.succeed(Pipeline, {
    stables: ["pipelineId", "accountId", "name", "createdAt"],

    diff: Effect.fn(function* ({ id, olds, news, output }) {
      const { accountId } = yield* yield* CloudflareEnvironment;
      if (!isResolved(news)) return undefined;
      if ((output?.accountId ?? accountId) !== accountId) {
        return { action: "replace" } as const;
      }
      const o = olds as PipelineProps | undefined;
      if (o === undefined) return undefined;
      // Pipelines have no update API — name and sql changes replace.
      const newName = yield* pipelineName(id, news.name);
      const oldName = output?.name ?? (yield* pipelineName(id, o.name));
      if (newName !== oldName) {
        return { action: "replace" } as const;
      }
      const oldSql = typeof o.sql === "string" ? o.sql : output?.sql;
      if (oldSql === undefined || canonicalSql(news.sql) === canonicalSql(oldSql)) {
        return oldSql === undefined ? undefined : ({ action: "noop" } as const);
      }
      // The replacement deletes the running pipeline — validate the new
      // statement first so bad SQL fails the plan, not the deploy. Tables
      // created in the same deploy are not visible yet, so a missing table
      // is not a validation failure.
      yield* pipelines.validateSqlPipeline({ accountId, sql: news.sql }).pipe(
        Effect.catchTags({
          TableNotFound: () => Effect.void,
          InvalidSql: (error) =>
            Effect.fail(
              new PipelineSqlInvalid({
                message: `Cloudflare.Pipelines.Pipeline '${id}': invalid SQL — ${error.message}`,
                sql: news.sql,
              }),
            ),
        }),
      );
      return { action: "replace" } as const;
    }),

    read: Effect.fn(function* ({ id, output, olds }) {
      const { accountId } = yield* yield* CloudflareEnvironment;
      const acct = output?.accountId ?? accountId;
      if (output?.pipelineId) {
        const observed = yield* getPipeline(acct, output.pipelineId);
        if (observed) return toAttributes(observed, acct);
      }
      // Cold read — pipeline names are unique per account; a
      // generated-name match is proof of ownership, an explicit name is
      // not.
      const name = yield* pipelineName(id, olds?.name);
      const match = yield* findPipelineByName(acct, name);
      if (match) {
        const attrs = toAttributes(match, acct);
        return olds?.name !== undefined ? Unowned(attrs) : attrs;
      }
      return undefined;
    }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const { accountId } = yield* yield* CloudflareEnvironment;
      const name = yield* pipelineName(id, news.name);
      const sql = news.sql as string;

      // 1. Observe — by cached id first, then by (unique) name so we
      //    recover from lost state writes.
      let observed = output?.pipelineId
        ? yield* getPipeline(output.accountId ?? accountId, output.pipelineId)
        : undefined;
      if (!observed) {
        observed = yield* findPipelineByName(accountId, name);
      }

      // Converge drift — there is no update API, so when the observed SQL
      // differs from the desired SQL (e.g. a referenced stream/sink was
      // replaced and `sql` was unresolved at diff time), delete the stale
      // pipeline and fall through to recreate it under the same name.
      // Nothing references a pipeline downstream, so this is safe.
      if (
        observed &&
        (canonicalSql(observed.sql) !== canonicalSql(sql) || observed.name !== name)
      ) {
        yield* deletePipeline(accountId, observed.id);
        // Wait until the delete is visible so the recreate below does not
        // race an `PipelineAlreadyExists` against the dying pipeline.
        yield* getPipeline(accountId, observed.id).pipe(
          Effect.repeat({
            schedule: Schedule.max([Schedule.exponential("250 millis"), Schedule.recurs(8)]),
            until: (p) => p === undefined,
          }),
        );
        observed = undefined;
      }

      // 2. Ensure — create when missing. The referenced stream/sink may
      //    have been created moments ago, so ride out eventual-consistency
      //    `TableNotFound` blips with a bounded retry. An AlreadyExists is
      //    a race or recovery — resolve it via the name lookup. There is
      //    no sync step: the SQL is immutable, so changes arrive as
      //    replacements (diff).
      if (!observed) {
        observed = yield* pipelines.createV1Pipeline({ accountId, name, sql }).pipe(
          Effect.retry({
            while: (e) => e._tag === "TableNotFound",
            schedule: Schedule.max([Schedule.exponential("500 millis"), Schedule.recurs(6)]),
          }),
          Effect.catchTag("PipelineAlreadyExists", (error) =>
            findPipelineByName(accountId, name).pipe(
              Effect.flatMap((match) => (match ? Effect.succeed(match) : Effect.fail(error))),
            ),
          ),
        );
      }

      return toAttributes(observed, accountId);
    }),

    delete: Effect.fn(function* ({ output }) {
      yield* deletePipeline(output.accountId, output.pipelineId);
    }),

    // Account collection — pipelines are account-scoped. Exhaustively
    // paginate the account-wide list and hydrate each row into the same
    // Attributes shape `read` returns.
    list: Effect.fn(function* () {
      const { accountId } = yield* yield* CloudflareEnvironment;
      return yield* pipelines.listV1Pipeline.pages({ accountId }).pipe(
        Stream.runCollect,
        Effect.map((chunk) =>
          Array.from(chunk).flatMap((page) =>
            (page.result ?? []).map((p) => toAttributes(p, accountId)),
          ),
        ),
      );
    }),
  });

/**
 * Canonical SQL for change detection: whitespace runs outside quoted
 * literals/identifiers collapse to one space, surrounding whitespace and
 * a trailing `;` are dropped.
 */
const canonicalSql = (sql: string): string => {
  let out = "";
  let quote: string | undefined;
  let pendingSpace = false;
  for (const ch of sql.trim().replace(/;\s*$/, "")) {
    if (quote) {
      out += ch;
      if (ch === quote) quote = undefined;
    } else if (/\s/.test(ch)) {
      pendingSpace = true;
    } else {
      if (pendingSpace && out.length > 0) out += " ";
      pendingSpace = false;
      if (ch === "'" || ch === '"' || ch === "`") quote = ch;
      out += ch;
    }
  }
  return out;
};

/**
 * The subset of pipeline state shared by get/list/create responses.
 */
interface ObservedPipeline {
  id: string;
  name: string;
  sql: string;
  status: string;
  createdAt: string;
  modifiedAt: string;
}

// Pipelines entity names must be alphanumeric/underscore only, so swap
// the default hyphen delimiter for underscores.
const pipelineName = (id: string, name: string | undefined) =>
  Effect.gen(function* () {
    if (name) return name;
    const generated = yield* createPhysicalName({
      id,
      lowercase: true,
      delimiter: "_",
    });
    return generated.replaceAll(/[^a-zA-Z0-9_]/g, "_");
  });

/**
 * Read a pipeline by id, mapping "gone" (`PipelineNotExists`, Cloudflare
 * error code 1000) to `undefined`.
 */
const getPipeline = (accountId: string, pipelineId: string) =>
  pipelines.getV1Pipeline({ accountId, pipelineId }).pipe(
    Effect.map((p): ObservedPipeline | undefined => p),
    Effect.catchTag("PipelineNotExists", () => Effect.undefined),
  );

/**
 * Idempotent delete — a pipeline that is already gone is success.
 */
const deletePipeline = (accountId: string, pipelineId: string) =>
  pipelines
    .deleteV1Pipeline({ accountId, pipelineId })
    .pipe(Effect.catchTag("PipelineNotExists", () => Effect.void));

const findPipelineByName = (accountId: string, name: string) =>
  pipelines.listV1Pipeline.items({ accountId }).pipe(
    Stream.runCollect,
    Effect.map((chunk): ObservedPipeline | undefined =>
      Array.from(chunk).find((p) => p.name === name),
    ),
  );

const toAttributes = (observed: ObservedPipeline, accountId: string): PipelineAttributes => ({
  pipelineId: observed.id,
  accountId,
  name: observed.name,
  sql: observed.sql,
  status: observed.status,
  createdAt: observed.createdAt,
  modifiedAt: observed.modifiedAt,
});