sailfork / t3code-github / .repos/ alchemy-effect/ packages/ cloudflare/ src/ Pipelines/ LegacyPipeline.ts

import { Credentials } from "@distilled.cloud/cloudflare/Credentials";
import * as pipelines from "@distilled.cloud/cloudflare/pipelines";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import * as Effect from "effect/Effect";
import type * as HttpClient from "effect/http/HttpClient";
import * as Predicate from "effect/Predicate";
import * as Redacted from "effect/Redacted";
import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
import type { Providers } from "../Providers.ts";

const TypeId = "Cloudflare.Pipelines.LegacyPipeline" as const;
type TypeId = typeof TypeId;

/**
 * HTTP ingest source — events are POSTed as JSON to the pipeline's
 * `endpoint` URL.
 */
export interface LegacyPipelineHttpSource {
  /** Accept events over HTTP at the pipeline endpoint. */
  type: "http";
  /**
   * Require Cloudflare API-token authentication on the ingest endpoint.
   * @default false
   */
  authentication?: boolean;
  /**
   * CORS configuration for browser-originated ingestion.
   */
  cors?: {
    /** Allowed origins, e.g. `["https://example.com"]` or `["*"]`. */
    origins?: string[];
  };
}

/**
 * Worker binding ingest source — events are sent from a Worker via a
 * `pipelines` binding.
 */
export interface LegacyPipelineBindingSource {
  /** Accept events from a Worker `pipelines` binding. */
  type: "binding";
}

/**
 * An ingest source of a legacy pipeline.
 */
export type LegacyPipelineSource = LegacyPipelineHttpSource | LegacyPipelineBindingSource;

/**
 * R2 destination configuration of a legacy pipeline.
 */
export interface LegacyPipelineDestination {
  /**
   * Name of the destination R2 bucket. The bucket must already exist.
   */
  bucket: string;
  /**
   * R2 S3-compatible credentials the pipeline uses to write objects.
   * Write-only — Cloudflare never echoes them back.
   */
  credentials: {
    /** R2 access key id (the API token id). */
    accessKeyId: Redacted.Redacted<string>;
    /** R2 secret access key (SHA-256 hex of the API token value). */
    secretAccessKey: Redacted.Redacted<string>;
    /**
     * S3-compatible endpoint of the R2 bucket.
     * @default https://{accountId}.r2.cloudflarestorage.com
     */
    endpoint?: string;
  };
  /**
   * Batching policy controlling when the pipeline flushes a batch of
   * events to R2.
   */
  batch?: {
    /** Flush once the batch reaches this size in bytes. */
    maxBytes?: number;
    /** Flush at most this many seconds after the batch was opened. */
    maxDurationS?: number;
    /** Flush once the batch reaches this many events. */
    maxRows?: number;
  };
  /**
   * Compression applied to output files.
   * @default "gzip"
   */
  compression?: "none" | "gzip" | "deflate";
  /**
   * Key prefix under which output objects are written.
   */
  prefix?: string;
  /**
   * Time-partitioned directory layout of output object keys
   * (strftime-style, e.g. `event_date=%F/hr=%H`).
   */
  filepath?: string;
  /**
   * Name template of output files within a partition.
   */
  filename?: string;
}

export interface LegacyPipelineProps {
  /**
   * Name of the pipeline. Unique per account; the legacy API addresses
   * pipelines by name, so changing it triggers a replacement. If
   * omitted, a unique name is generated from the app, stage, and
   * logical ID.
   * @default ${app}-${id}-${stage}-${suffix}
   */
  name?: string;
  /**
   * Ingest sources accepted by the pipeline.
   * @default [{ type: "http" }, { type: "binding" }]
   */
  source?: LegacyPipelineSource[];
  /**
   * R2 destination the pipeline batches events into.
   */
  destination: LegacyPipelineDestination;
}

export interface LegacyPipelineAttributes {
  /** Cloudflare-assigned pipeline identifier. */
  pipelineId: string;
  /** Account that owns the pipeline. */
  accountId: string;
  /** Pipeline name (unique per account; the API identifier). */
  name: string;
  /** HTTP endpoint URL events can be POSTed to. */
  endpoint: string;
  /** Destination R2 bucket name. */
  bucket: string;
  /** Version number of the last saved configuration. */
  version: number;
}

export type LegacyPipeline = Resource<
  TypeId,
  LegacyPipelineProps,
  LegacyPipelineAttributes,
  never,
  Providers
>;

/**
 * A **legacy** Cloudflare Pipeline — the original HTTP-ingest → R2 batch
 * product (`/accounts/{account}/pipelines`).
 *
 * :::caution
 * This is the **deprecated, legacy** Pipelines API. Cloudflare has
 * superseded it with the SQL-based product — prefer
 * {@link Stream}, {@link Sink}, and {@link Pipeline}
 * for new infrastructure. This resource exists only to manage
 * pre-existing legacy pipelines.
 * :::
 *
 * A legacy pipeline accepts JSON events over HTTP (and/or a Worker
 * `pipelines` binding) and batches them into an R2 bucket using
 * S3-compatible credentials.
 * ### Creating a Legacy Pipeline
 * **Example:** HTTP ingest into R2
 * The S3-compatible credentials are derived from a Cloudflare API token:
 * the access key id is the token id and the secret is the SHA-256 hex
 * digest of the token value.
 * ```typescript
 * const bucket = yield* Cloudflare.R2.Bucket("events", {});
 *
 * const pipeline = yield* Cloudflare.Pipelines.LegacyPipeline("ingest", {
 *   destination: {
 *     bucket: bucket.bucketName,
 *     credentials: {
 *       accessKeyId: yield* Config.Redacted("R2_ACCESS_KEY_ID"),
 *       secretAccessKey: yield* Config.Redacted("R2_SECRET_ACCESS_KEY"),
 *     },
 *   },
 * });
 * // POST events to pipeline.endpoint
 * ```
 *
 * **Example:** Tuned batching and CORS
 * ```typescript
 * const pipeline = yield* Cloudflare.Pipelines.LegacyPipeline("ingest", {
 *   source: [
 *     { type: "http", cors: { origins: ["https://example.com"] } },
 *   ],
 *   destination: {
 *     bucket: bucket.bucketName,
 *     credentials,
 *     batch: { maxDurationS: 10, maxRows: 1000 },
 *     compression: "gzip",
 *     prefix: "ingest",
 *   },
 * });
 * ```
 *
 * @see https://developers.cloudflare.com/pipelines/
 *
 * @resource
 * @product Pipelines
 * @category Storage & Databases
 */
export const LegacyPipeline = Resource<LegacyPipeline>(TypeId);

/**
 * Returns true if the given value is a LegacyPipeline resource.
 */
export const isLegacyPipeline = (value: unknown): value is LegacyPipeline =>
  Predicate.hasProperty(value, "Type") && value.Type === TypeId;

export const LegacyPipelineProvider = () =>
  Provider.succeed(LegacyPipeline, {
    stables: ["pipelineId", "accountId", "name", "endpoint"],

    diff: Effect.fn(function* ({ id, olds, news, output }) {
      const { accountId } = yield* yield* CloudflareEnvironment;
      if (!isResolved(news)) return undefined;
      if ((output?.accountId ?? accountId) !== accountId) {
        return { action: "replace" } as const;
      }
      const o = olds as LegacyPipelineProps | undefined;
      if (o === undefined) return undefined;
      // The legacy API addresses pipelines by name — a name change is a
      // replacement. Everything else updates in place via PUT.
      const newName = yield* legacyPipelineName(id, news.name);
      const oldName = output?.name ?? (yield* legacyPipelineName(id, o.name));
      if (newName !== oldName) {
        return { action: "replace" } as const;
      }
      return undefined;
    }),

    list: Effect.fn(function* () {
      const { accountId } = yield* yield* CloudflareEnvironment;
      // The list endpoint returns summary items (id/name/endpoint only),
      // so hydrate each by name into the exact `read` Attributes shape
      // with bounded concurrency and a typed per-item not-found skip
      // (a pipeline can vanish between the list and the get).
      const summaries = yield* listLegacyPipelineSummaries(accountId);
      const rows = yield* Effect.forEach(
        summaries,
        (summary) =>
          getLegacyPipeline(accountId, summary.name).pipe(
            Effect.map((observed) => (observed ? toAttributes(observed, accountId) : undefined)),
          ),
        { concurrency: 10 },
      );
      return rows.filter((row): row is LegacyPipelineAttributes => row !== undefined);
    }),

    read: Effect.fn(function* ({ id, output, olds }) {
      const { accountId } = yield* yield* CloudflareEnvironment;
      const acct = output?.accountId ?? accountId;
      if (output?.name) {
        const observed = yield* getLegacyPipeline(acct, output.name);
        if (observed) return toAttributes(observed, acct);
      }
      // Cold read — names are unique per account; a generated-name match
      // is proof of ownership, an explicit name is not.
      const name = yield* legacyPipelineName(id, olds?.name);
      const observed = yield* getLegacyPipeline(acct, name);
      if (observed) {
        const attrs = toAttributes(observed, acct);
        return olds?.name !== undefined ? Unowned(attrs) : attrs;
      }
      return undefined;
    }),

    reconcile: Effect.fn(function* ({ id, news, olds, output }) {
      const { accountId } = yield* yield* CloudflareEnvironment;
      const name = yield* legacyPipelineName(id, news.name);

      // 1. Observe — the name is the API identifier, so a single get
      //    covers both the cached-output and cold-recovery cases.
      let observed = yield* getLegacyPipeline(output?.accountId ?? accountId, output?.name ?? name);

      // 2. Ensure — create when missing.
      if (!observed) {
        observed = yield* pipelines.createPipeline({
          accountId,
          name,
          source: toRequestSource(news.source),
          destination: toRequestDestination(accountId, news.destination),
        });
        return toAttributes(observed, accountId);
      }

      // 3. Sync — diff observed config against desired and PUT the full
      //    desired state only when something changed. Credentials are
      //    write-only, so `olds` is the only (best-effort) baseline for
      //    them: on adoption (`olds` undefined) or a credential change we
      //    push an update unconditionally.
      const credsRotated =
        olds === undefined ||
        Redacted.value(news.destination.credentials.accessKeyId) !==
          Redacted.value(olds.destination.credentials.accessKeyId) ||
        Redacted.value(news.destination.credentials.secretAccessKey) !==
          Redacted.value(olds.destination.credentials.secretAccessKey);
      if (credsRotated || drifted(observed, news)) {
        observed = yield* pipelines.updatePipeline({
          accountId,
          pipelineName: observed.name,
          name,
          source: toRequestSource(news.source),
          destination: toRequestDestination(accountId, news.destination),
        });
      }

      return toAttributes(observed, accountId);
    }),

    delete: Effect.fn(function* ({ output }) {
      yield* pipelines
        .deletePipeline({
          accountId: output.accountId,
          pipelineName: output.name,
        })
        .pipe(Effect.catchTag("PipelineNotExists", () => Effect.void));
    }),
  });

/**
 * The pipeline state shared by get/create/update responses.
 */
interface ObservedLegacyPipeline {
  id: string;
  name: string;
  endpoint: string;
  version: number;
  destination: {
    batch: { maxBytes: number; maxDurationS: number; maxRows: number };
    compression: { type: string };
    path: {
      bucket: string;
      filename?: string | null;
      filepath?: string | null;
      prefix?: string | null;
    };
  };
  source: readonly {
    type: string;
    authentication?: boolean | null;
    cors?: { origins?: readonly string[] | null } | null;
  }[];
}

const legacyPipelineName = (id: string, name: string | undefined) =>
  Effect.gen(function* () {
    if (name) return name;
    return yield* createPhysicalName({ id, lowercase: true, delimiter: "-" });
  });

/**
 * Read a legacy pipeline by name, mapping "gone" (`PipelineNotExists`,
 * Cloudflare error code 1000) to `undefined`.
 */
const getLegacyPipeline = (accountId: string, pipelineName: string) =>
  pipelines.getPipeline({ accountId, pipelineName }).pipe(
    Effect.map((p): ObservedLegacyPipeline | undefined => p),
    Effect.catchTag("PipelineNotExists", () => Effect.undefined),
  );

interface LegacyPipelineSummary {
  id: string;
  name: string;
  endpoint: string;
}

/**
 * Enumerate every legacy pipeline summary in an account. The distilled
 * `listPipelines` op is not stream-paginated, so walk the `page`/
 * `per_page` query params using `result_info.total_count` (falling back
 * to a short-page sentinel) until every page is collected. The list
 * endpoint caps `per_page` at 50 and returns only summary fields.
 */
const listLegacyPipelineSummaries = (accountId: string) => {
  const perPage = 50;
  const collect = (
    page: number,
    acc: LegacyPipelineSummary[],
  ): Effect.Effect<
    LegacyPipelineSummary[],
    pipelines.ListPipelinesError,
    Credentials | HttpClient.HttpClient
  > =>
    Effect.gen(function* () {
      const response = yield* pipelines.listPipelines({
        accountId,
        page: String(page),
        perPage: String(perPage),
      });
      const results = (response.results ?? []).map((p): LegacyPipelineSummary => ({
        id: p.id,
        name: p.name ?? "",
        endpoint: p.endpoint ?? "",
      }));
      const next = [...acc, ...results];
      const total = response.resultInfo?.totalCount;
      const done = results.length < perPage || (total != null && next.length >= total);
      return done ? next : yield* collect(page + 1, next);
    });
  return collect(1, []);
};

const defaultSources: LegacyPipelineSource[] = [{ type: "http" }, { type: "binding" }];

const toRequestSource = (source: LegacyPipelineSource[] | undefined) =>
  (source ?? defaultSources).map((s) =>
    s.type === "http"
      ? {
          format: "json" as const,
          type: s.type,
          authentication: s.authentication,
          cors: s.cors,
        }
      : { format: "json" as const, type: s.type },
  );

const toRequestDestination = (accountId: string, destination: LegacyPipelineDestination) => ({
  type: "r2" as const,
  format: "json" as const,
  batch: destination.batch ?? {},
  compression: { type: destination.compression },
  credentials: {
    accessKeyId: Redacted.value(destination.credentials.accessKeyId),
    secretAccessKey: Redacted.value(destination.credentials.secretAccessKey),
    endpoint: destination.credentials.endpoint ?? `https://${accountId}.r2.cloudflarestorage.com`,
  },
  path: {
    bucket: destination.bucket as string,
    prefix: destination.prefix,
    filepath: destination.filepath,
    filename: destination.filename,
  },
});

/**
 * Detect drift between echoed cloud config and desired props. Only
 * user-declared optional fields are diffed so we don't fight server-side
 * defaults; credentials are write-only and handled separately.
 */
const drifted = (observed: ObservedLegacyPipeline, news: LegacyPipelineProps): boolean => {
  const d = news.destination;
  const path = observed.destination.path;
  if (path.bucket !== (d.bucket as string)) return true;
  if (d.prefix !== undefined && d.prefix !== (path.prefix ?? undefined)) {
    return true;
  }
  if (d.filepath !== undefined && d.filepath !== (path.filepath ?? undefined)) {
    return true;
  }
  if (d.filename !== undefined && d.filename !== (path.filename ?? undefined)) {
    return true;
  }
  if (d.compression !== undefined && d.compression !== observed.destination.compression.type) {
    return true;
  }
  const batch = observed.destination.batch;
  if (d.batch?.maxBytes !== undefined && d.batch.maxBytes !== batch.maxBytes) {
    return true;
  }
  if (d.batch?.maxDurationS !== undefined && d.batch.maxDurationS !== batch.maxDurationS) {
    return true;
  }
  if (d.batch?.maxRows !== undefined && d.batch.maxRows !== batch.maxRows) {
    return true;
  }
  // Sources — compare the declared set of source types and the declared
  // http options against what Cloudflare echoes.
  const desired = news.source ?? defaultSources;
  const observedTypes = observed.source.map((s) => s.type).sort();
  const desiredTypes = desired.map((s) => s.type).sort();
  if (observedTypes.join(",") !== desiredTypes.join(",")) return true;
  const desiredHttp = desired.find((s): s is LegacyPipelineHttpSource => s.type === "http");
  const observedHttp = observed.source.find((s) => s.type === "http");
  if (desiredHttp && observedHttp) {
    if (
      desiredHttp.authentication !== undefined &&
      desiredHttp.authentication !== (observedHttp.authentication ?? false)
    ) {
      return true;
    }
    if (desiredHttp.cors?.origins !== undefined) {
      const observedOrigins = observedHttp.cors?.origins ?? [];
      if (
        desiredHttp.cors.origins.length !== observedOrigins.length ||
        desiredHttp.cors.origins.some((o, i) => o !== observedOrigins[i])
      ) {
        return true;
      }
    }
  }
  return false;
};

const toAttributes = (
  observed: ObservedLegacyPipeline,
  accountId: string,
): LegacyPipelineAttributes => ({
  pipelineId: observed.id,
  accountId,
  name: observed.name,
  endpoint: observed.endpoint,
  bucket: observed.destination.path.bucket,
  version: observed.version,
});