sailfork / t3code-github / .repos/ alchemy-effect/ packages/ cloudflare/ src/ Pipelines/ Stream.ts

import * as pipelines from "@distilled.cloud/cloudflare/pipelines";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import type { PropsInput } from "alchemy/Input";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource, type ResourceClass } from "alchemy/Resource";
import * as Effect from "effect/Effect";
import * as Effectable from "effect/Effectable";
import * as Predicate from "effect/Predicate";
import * as Schedule from "effect/Schedule";
import type * as Schema from "effect/Schema";
import * as EffectStream from "effect/Stream";
import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
import type { Providers } from "../Providers.ts";
import {
  canonicalStreamFields,
  isEffectSchema,
  StreamSchemaUnsupported,
  streamFieldsFromSchema,
  type StreamFieldsSchema,
} from "./StreamSchema.ts";

const StreamTypeId = "Cloudflare.Pipelines.Stream" as const;
type StreamTypeId = typeof StreamTypeId;

/**
 * Input format of events ingested by the stream.
 */
export interface StreamFormat {
  /** Only JSON ingestion is supported. */
  type: "json";
  /**
   * Accept events of any shape (no schema enforcement).
   */
  unstructured?: boolean;
  /**
   * How timestamps are rendered in ingested JSON.
   * @default "rfc3339"
   */
  timestampFormat?: "rfc3339" | "unix_millis";
  /** How decimals are encoded in ingested JSON. */
  decimalEncoding?: "number" | "string" | "bytes";
}

/**
 * HTTP ingest endpoint configuration. Mutable in place.
 */
export interface StreamHttp {
  /**
   * Whether the stream exposes an HTTP ingest endpoint.
   * @default true
   */
  enabled?: boolean;
  /**
   * Whether requests to the HTTP endpoint must carry a Cloudflare API
   * token with the `Pipelines Send` permission.
   * @default false
   */
  authentication?: boolean;
  /** CORS configuration for browser-originated ingestion. */
  cors?: {
    /** Allowed origins, e.g. `["https://app.example.com"]` or `["*"]`. */
    origins?: string[];
  };
}

export interface StreamProps {
  /**
   * Name of the stream. Unique per account; must be alphanumeric and
   * underscores only (it is referenced as a SQL table name). If omitted,
   * a unique name is generated from the app, stage, and logical ID.
   * Changing the name triggers a replacement.
   * @default ${app}_${id}_${stage}_${suffix}
   */
  name?: string;
  /**
   * Structured schema of ingested events — either an explicit field list
   * or an Effect Schema (`Schema.Struct` / `Schema.Class`), which is
   * converted to the field list and also types and encodes the records
   * sent through `WriteStream` / `StreamSink`.
   *
   * Immutable — changing the (derived) field list triggers a replacement,
   * which drops buffered events and dependent pipelines. Equivalent
   * spellings (a field list ⇄ the matching Effect Schema) are a no-op.
   * When omitted, the stream accepts unstructured JSON events.
   */
  schema?: StreamFieldsSchema | Schema.Top;
  /**
   * Input format configuration. Immutable — changing it triggers a
   * replacement.
   * @default { type: "json" }
   */
  format?: StreamFormat;
  /**
   * HTTP ingest endpoint configuration. Mutable in place.
   *
   * - omitted / `false` — no public HTTP endpoint (`{ enabled: false }`)
   * - `true` — endpoint enabled, API-token authentication required
   * - an object — explicit settings
   * @default false
   */
  http?: boolean | StreamHttp;
  /**
   * Whether Workers can send events to this stream via a `pipelines`
   * binding. Mutable in place.
   * @default { enabled: true }
   */
  workerBinding?: {
    /** Whether the Worker binding is enabled. */
    enabled: boolean;
  };
}

export interface StreamAttributes {
  /** Cloudflare-assigned stream identifier. */
  streamId: string;
  /** Account that owns the stream. */
  accountId: string;
  /** Stream name (unique per account). */
  name: string;
  /** HTTP ingest endpoint URL, when HTTP ingestion is enabled. */
  endpoint: string | undefined;
  /** Whether the HTTP ingest endpoint is enabled. */
  httpEnabled: boolean;
  /** Whether the HTTP ingest endpoint requires authentication. */
  httpAuthentication: boolean;
  /** Allowed CORS origins of the HTTP ingest endpoint. */
  corsOrigins: string[] | undefined;
  /** Whether Workers can send events via a `pipelines` binding. */
  workerBindingEnabled: boolean;
  /** Current version of the stream. */
  version: number;
  /** When the stream was created. */
  createdAt: string;
  /** When the stream was last modified. */
  modifiedAt: string;
}

/**
 * A Pipelines stream. `A` is the record type sent through `WriteStream` /
 * `StreamSink` — the Effect Schema's `Type` when the stream was declared
 * with one.
 */
export interface Stream<A = unknown> extends Resource<
  StreamTypeId,
  StreamProps,
  StreamAttributes,
  never,
  Providers
> {
  /**
   * The Effect Schema the stream was declared with, if any. Used by the
   * producer clients to encode records; never persisted.
   */
  readonly RecordSchema: Schema.Top | undefined;
  /** Phantom record type. */
  readonly "~record"?: A;
}

/**
 * The record type a producer sends to a `Stream<A>`: the declared Effect
 * Schema's `Type`, or an untyped JSON object.
 */
export type StreamRecord<A> = unknown extends A ? Record<string, unknown> : A;

/**
 * Props of a stream declared with an Effect Schema.
 */
export type StreamSchemaProps<S extends Schema.Top> = PropsInput<Omit<StreamProps, "schema">> & {
  schema: S;
};

const StreamResource = Resource<Stream>(StreamTypeId);

/**
 * The Stream constructor: the plain resource constructor plus an overload
 * that infers the record type from an Effect Schema `schema`.
 */
export type StreamClass = {
  <const S extends Schema.Top>(
    id: string,
    props: StreamSchemaProps<S>,
  ): Effect.Effect<Stream<S["Type"]>, never, Providers>;
} & ResourceClass<Stream>;

/**
 * Convert an Effect Schema `schema` prop into the persisted field list and
 * keep the schema aside for the producer clients. Dies with a clear
 * {@link StreamSchemaUnsupported} when the schema has no stream
 * representation — this runs while the program is constructed, before
 * anything is planned.
 */
const normalizeStreamProps = (
  props: unknown,
): Effect.Effect<{ props: unknown; recordSchema: Schema.Top | undefined }> =>
  Effect.suspend((): Effect.Effect<{ props: unknown; recordSchema: Schema.Top | undefined }> => {
    const schema = (props as { schema?: unknown } | undefined)?.schema;
    if (!isEffectSchema(schema)) {
      return Effect.succeed({ props, recordSchema: undefined });
    }
    return resolveSchema(schema).pipe(
      Effect.map((resolved) => ({
        props: { ...(props as object), schema: resolved },
        recordSchema: schema,
      })),
      Effect.orDie,
    );
  });

const constructStream = (id: string, props: unknown) =>
  Effect.gen(function* () {
    let recordSchema: Schema.Top | undefined;
    const capture = (n: { props: unknown; recordSchema: Schema.Top | undefined }) => {
      recordSchema = n.recordSchema;
      return n.props;
    };
    const normalized = Effect.isEffect(props)
      ? // oxlint-disable-next-line effecttsgo/unsafe-effect-type-assertion -- props are untyped; Effect-valued props are resolved by the engine
        (props as Effect.Effect<unknown>).pipe(
          Effect.flatMap(normalizeStreamProps),
          Effect.map(capture),
        )
      : capture(yield* normalizeStreamProps(props));
    const construct = StreamResource as unknown as (
      id: string,
      props: unknown,
    ) => Effect.Effect<Stream, never, Providers>;
    const stream = yield* construct(id, normalized);
    (stream as { RecordSchema: Schema.Top | undefined }).RecordSchema = recordSchema;
    return stream;
  });

/**
 * A Cloudflare Pipelines stream — the ingestion endpoint of the Pipelines
 * product (also exported as `Cloudflare.Basin.Stream`). Events are sent to
 * a stream from Workers (`WriteStream` / `StreamSink`) or over HTTP,
 * transformed by a SQL {@link Pipeline}, and written to a {@link Sink}.
 *
 * The stream's `schema` and `format` are fixed at creation (changing them
 * triggers a replacement, which drops buffered events and dependent
 * pipelines); the HTTP endpoint and Worker-binding toggles are mutable in
 * place.
 *
 * Pipelines is ingest-only — there is no event source on a stream. To
 * process data after it lands, subscribe to the sink's bucket with
 * `Cloudflare.R2.BucketEventNotification`.
 *
 * Cloudflare accepts records that violate a structured stream's schema
 * (they are dropped later, during processing). Declare the schema as an
 * Effect Schema to have the producer clients validate and encode every
 * record before it is sent.
 *
 * :::caution
 * A stream declared without `http` has **no** public ingest endpoint
 * (`http: { enabled: false }`). Earlier versions sent nothing, and
 * Cloudflare's default is a public, unauthenticated endpoint. Existing
 * streams are updated in place on the next deploy; set `http: true`
 * (authenticated) or `http: { enabled: true }` to keep an endpoint.
 * :::
 *
 * ### Creating a Stream
 * **Example:** Unstructured stream with default settings
 * ```typescript
 * const stream = yield* Cloudflare.Basin.Stream("events", {});
 * ```
 *
 * **Example:** Structured stream from an Effect Schema
 * ```typescript
 * class PageView extends Schema.Class<PageView>("PageView")({
 *   url: Schema.String,
 *   at: Schema.Date,
 *   tags: Schema.Array(Schema.String),
 *   user: Schema.optional(Schema.Struct({ id: Schema.String })),
 * }) {}
 *
 * // Stream<PageView>: WriteStream / StreamSink are typed and encode records
 * const stream = yield* Cloudflare.Basin.Stream("PageViews", {
 *   schema: PageView,
 * });
 * ```
 *
 * **Example:** Structured stream with an explicit field list
 * ```typescript
 * const stream = yield* Cloudflare.Basin.Stream("clicks", {
 *   schema: {
 *     fields: [
 *       { type: "string", name: "url", required: true },
 *       { type: "timestamp", name: "ts", unit: "millisecond" },
 *       { type: "list", name: "tags", items: { type: "string" } },
 *       {
 *         type: "struct",
 *         name: "user",
 *         fields: [{ type: "string", name: "id" }],
 *       },
 *     ],
 *   },
 * });
 * ```
 *
 * ### HTTP ingestion
 * **Example:** Authenticated endpoint
 * ```typescript
 * const stream = yield* Cloudflare.Basin.Stream("events", { http: true });
 * // POST a JSON array to stream.endpoint with a `Pipelines Send` token,
 * // or use Cloudflare.Basin.WriteStreamHttp / WriteStreamLocal.
 * ```
 *
 * **Example:** Public endpoint with CORS
 * ```typescript
 * const stream = yield* Cloudflare.Basin.Stream("beacons", {
 *   http: {
 *     enabled: true,
 *     authentication: false,
 *     cors: { origins: ["https://app.example.com"] },
 *   },
 * });
 * ```
 *
 * ### Wiring into a Pipeline
 * **Example:** Stream → SQL Pipeline → Sink
 * ```typescript
 * const pipeline = yield* Cloudflare.Basin.Pipeline("etl", {
 *   sql: Output.interpolate`INSERT INTO ${sink.name} SELECT * FROM ${stream.name}`,
 * });
 * ```
 *
 * @see https://developers.cloudflare.com/pipelines/
 *
 * @resource
 * @product Pipelines
 * @category Storage & Databases
 */
export const Stream: StreamClass = Object.assign(
  (...args: [id: string, props?: unknown] | [methods: object]) =>
    typeof args[0] === "object"
      ? Object.assign(Stream, args[0])
      : constructStream(args[0], (args as [string, unknown])[1]),
  StreamResource,
  Effectable.Prototype({
    label: `Resource<${StreamTypeId}>`,
    evaluate: () => Effect.succeed((id: string, props: unknown) => constructStream(id, props)),
  }),
) as unknown as StreamClass;

/**
 * Returns true if the given value is a Stream resource.
 */
export const isStream = (value: unknown): value is Stream =>
  Predicate.hasProperty(value, "Type") && value.Type === StreamTypeId;

export const StreamProvider = () =>
  Provider.succeed(Stream, {
    stables: ["streamId", "accountId", "name", "createdAt"],

    diff: Effect.fn(function* ({ id, olds, news, output }) {
      const { accountId } = yield* yield* CloudflareEnvironment;
      if (!isResolved(news)) return undefined;
      if ((output?.accountId ?? accountId) !== accountId) {
        return { action: "replace" } as const;
      }
      const o = olds as StreamProps | undefined;
      const newName = yield* streamName(id, news.name);
      const oldName = output?.name ?? (yield* streamName(id, o?.name));
      if (newName !== oldName) {
        return { action: "replace" } as const;
      }
      if (o === undefined) return undefined;
      // Schema and format are immutable. Compare CANONICAL derived values
      // so equivalent spellings (field list ⇄ Effect Schema, omitted ⇄
      // explicit defaults) never replace.
      const newSchema = yield* canonicalSchema(news.schema);
      const oldSchema = yield* canonicalSchema(o.schema);
      if (!stableEquals(newSchema, oldSchema)) {
        return { action: "replace" } as const;
      }
      if (!stableEquals(canonicalFormat(news.format), canonicalFormat(o.format))) {
        return { action: "replace" } as const;
      }
      if (output === undefined) return undefined;
      // http/workerBinding are patchable: diff desired against the last
      // observed state, so the secure http default reaches streams created
      // before it existed.
      const httpConverged = httpMatches(desiredHttp(news.http), {
        enabled: output.httpEnabled,
        authentication: output.httpAuthentication,
        origins: output.corsOrigins,
      });
      const bindingConverged =
        news.workerBinding === undefined ||
        news.workerBinding.enabled === output.workerBindingEnabled;
      return httpConverged && bindingConverged
        ? ({ action: "noop" } as const)
        : ({ action: "update" } as const);
    }),

    read: Effect.fn(function* ({ id, output, olds }) {
      const { accountId } = yield* yield* CloudflareEnvironment;
      const acct = output?.accountId ?? accountId;
      if (output?.streamId) {
        const observed = yield* getStream(acct, output.streamId);
        if (observed) return toAttributes(observed, acct);
      }
      // Cold read — stream names are unique per account, so an exact name
      // match identifies the resource. A generated name embeds the
      // instance id (proof of ownership); a user-provided name does not,
      // so gate takeover behind the adopt policy.
      const name = yield* streamName(id, olds?.name);
      const match = yield* findStreamByName(acct, name);
      if (match) {
        const attrs = toAttributes(match, acct);
        return olds?.name !== undefined ? Unowned(attrs) : attrs;
      }
      return undefined;
    }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const { accountId } = yield* yield* CloudflareEnvironment;
      const name = yield* streamName(id, news.name);
      const http = desiredHttp(news.http);

      // 1. Observe — the cached streamId is a hint, not a guarantee; a
      //    missing stream falls through to a name lookup and then create.
      let observed = output?.streamId
        ? yield* getStream(output.accountId ?? accountId, output.streamId)
        : undefined;
      if (!observed) {
        observed = yield* findStreamByName(accountId, name);
      }

      // 2. Ensure — create when missing. Stream names are unique, so an
      //    AlreadyExists is a race (or recovery from a lost state write):
      //    fall back to the name lookup.
      if (!observed) {
        const schema = yield* resolveSchema(news.schema);
        observed = yield* pipelines
          .createStream({
            accountId,
            name,
            format: news.format,
            // The distilled field union does not model nested `items` /
            // `fields`; they pass through to the wire verbatim.
            schema: schema as pipelines.CreateStreamRequest["schema"],
            http,
            workerBinding: news.workerBinding,
          })
          .pipe(
            Effect.catchTag("StreamAlreadyExists", (error) =>
              findStreamByName(accountId, name).pipe(
                Effect.flatMap((match) => (match ? Effect.succeed(match) : Effect.fail(error))),
              ),
            ),
          );
      }

      // 3. Sync — http and workerBinding are the only mutable aspects.
      //    Diff observed cloud state against the declared desired state
      //    and PATCH only the delta; skip the API entirely on a no-op.
      const patch: pipelines.PatchStreamRequest = {
        accountId,
        streamId: observed.id,
      };
      let dirty = false;
      if (
        !httpMatches(http, {
          enabled: observed.http.enabled,
          authentication: observed.http.authentication,
          origins: observed.http.cors?.origins,
        })
      ) {
        patch.http = http;
        dirty = true;
      }
      if (
        news.workerBinding !== undefined &&
        news.workerBinding.enabled !== observed.workerBinding.enabled
      ) {
        patch.workerBinding = { enabled: news.workerBinding.enabled };
        dirty = true;
      }
      if (dirty) {
        observed = yield* pipelines.patchStream(patch);
      }

      return toAttributes(observed, accountId);
    }),

    delete: Effect.fn(function* ({ output }) {
      // A dependent pipeline's own deletion may still be propagating —
      // ride out `StreamInUse` (HTTP 422 "still in use") with a bounded
      // retry. A stream that is already gone is success.
      yield* pipelines
        .deleteStream({
          accountId: output.accountId,
          streamId: output.streamId,
        })
        .pipe(
          Effect.retry({
            while: (e) => e._tag === "StreamInUse",
            schedule: Schedule.max([Schedule.exponential("500 millis"), Schedule.recurs(8)]),
          }),
          Effect.catchTags({
            StreamNotFound: () => Effect.void,
            InvalidStreamId: () => Effect.void,
          }),
        );
    }),

    // Account-scoped collection: exhaustively paginate every stream in the
    // account and hydrate each into the same Attributes shape `read`
    // returns. The list item shape is structurally identical to the
    // get/create response consumed by `toAttributes`.
    list: Effect.fn(function* () {
      const { accountId } = yield* yield* CloudflareEnvironment;
      return yield* pipelines.listStreams.pages({ accountId }).pipe(
        EffectStream.runCollect,
        Effect.map((chunk) =>
          Array.from(chunk).flatMap((page) =>
            (page.result ?? []).map((s) => toAttributes(s, accountId)),
          ),
        ),
      );
    }),
  });

/**
 * The subset of stream state shared by get/list/create/patch responses.
 */
interface ObservedStream {
  id: string;
  name: string;
  endpoint?: string | null;
  http: {
    enabled: boolean;
    authentication: boolean;
    cors?: { origins?: readonly string[] | null } | null;
  };
  workerBinding: { enabled: boolean };
  version: number;
  createdAt: string;
  modifiedAt: string;
}

// Pipelines entity names must be alphanumeric/underscore only (they are
// referenced as SQL table names), so swap the default hyphen delimiter
// for underscores.
const streamName = (id: string, name: string | undefined) =>
  Effect.gen(function* () {
    if (name) return name;
    const generated = yield* createPhysicalName({
      id,
      lowercase: true,
      delimiter: "_",
    });
    return generated.replaceAll(/[^a-zA-Z0-9_]/g, "_");
  });

/**
 * Read a stream by id, mapping "gone" (`StreamNotFound`, Cloudflare error
 * code 1016, or `InvalidStreamId` for a malformed/foreign id) to
 * `undefined`.
 */
const getStream = (accountId: string, streamId: string) =>
  pipelines.getStream({ accountId, streamId }).pipe(
    Effect.map((s): ObservedStream | undefined => s),
    Effect.catchTags({
      StreamNotFound: () => Effect.undefined,
      InvalidStreamId: () => Effect.undefined,
    }),
  );

const findStreamByName = (accountId: string, name: string) =>
  pipelines.listStreams.items({ accountId }).pipe(
    EffectStream.runCollect,
    Effect.map((chunk): ObservedStream | undefined =>
      Array.from(chunk).find((s) => s.name === name),
    ),
  );

/** Resolve a `schema` prop (field list or Effect Schema) to the field-list form. */
const resolveSchema = (
  schema: StreamFieldsSchema | Schema.Top | undefined,
): Effect.Effect<StreamFieldsSchema | undefined, StreamSchemaUnsupported> =>
  isEffectSchema(schema)
    ? Effect.try({
        try: () => ({ fields: streamFieldsFromSchema(schema) }),
        catch: (error) =>
          error instanceof StreamSchemaUnsupported
            ? error
            : new StreamSchemaUnsupported({ path: "", message: String(error) }),
      })
    : Effect.succeed(schema as StreamFieldsSchema | undefined);

const canonicalSchema = (schema: StreamFieldsSchema | Schema.Top | undefined) =>
  resolveSchema(schema).pipe(
    Effect.map((resolved) =>
      resolved === undefined ? undefined : canonicalStreamFields(resolved.fields),
    ),
  );

const canonicalFormat = (format: StreamFormat | undefined) => {
  const f = format ?? { type: "json" as const };
  return {
    type: f.type,
    unstructured: f.unstructured,
    timestampFormat: f.timestampFormat ?? "rfc3339",
    decimalEncoding: f.decimalEncoding,
  };
};

interface DesiredHttp {
  enabled: boolean;
  authentication: boolean;
  cors?: { origins: string[] };
}

/**
 * Desired HTTP ingest settings. Omitted (or `false`) means no endpoint —
 * Cloudflare's own default is a public, unauthenticated endpoint.
 */
const desiredHttp = (http: boolean | StreamHttp | undefined): DesiredHttp => {
  if (http === undefined || http === false) {
    return { enabled: false, authentication: false };
  }
  if (http === true) return { enabled: true, authentication: true };
  return {
    enabled: http.enabled ?? true,
    authentication: http.authentication ?? false,
    cors: http.cors?.origins ? { origins: http.cors.origins } : undefined,
  };
};

const httpMatches = (
  desired: DesiredHttp,
  observed: {
    enabled: boolean;
    authentication: boolean;
    origins: readonly string[] | null | undefined;
  },
) => {
  // A disabled endpoint's auth/CORS settings are inert.
  if (!desired.enabled) return !observed.enabled;
  return (
    observed.enabled &&
    desired.authentication === observed.authentication &&
    sameOrigins(desired.cors?.origins, observed.origins)
  );
};

const sameOrigins = (
  desired: readonly string[] | undefined,
  observed: readonly string[] | null | undefined,
) => {
  // Only diff origins when the user declared them; Cloudflare's default
  // is not ours to fight.
  if (desired === undefined) return true;
  const have = observed ?? [];
  return (
    desired.length === have.length && [...desired].sort().join(",") === [...have].sort().join(",")
  );
};

/**
 * Key-order-insensitive structural equality for plain JSON-ish prop
 * values (schemas, formats).
 */
const stableEquals = (a: unknown, b: unknown): boolean => stableStringify(a) === stableStringify(b);

const stableStringify = (value: unknown): string =>
  JSON.stringify(value, (_key, v) =>
    v !== null && typeof v === "object" && !Array.isArray(v)
      ? Object.fromEntries(
          Object.entries(v as Record<string, unknown>).sort(([x], [y]) => x.localeCompare(y)),
        )
      : v,
  ) ?? "undefined";

const toAttributes = (observed: ObservedStream, accountId: string): StreamAttributes => ({
  streamId: observed.id,
  accountId,
  name: observed.name,
  endpoint: observed.endpoint ?? undefined,
  httpEnabled: observed.http.enabled,
  httpAuthentication: observed.http.authentication,
  corsOrigins: observed.http.cors?.origins ? [...observed.http.cors.origins] : undefined,
  workerBindingEnabled: observed.workerBinding.enabled,
  version: observed.version,
  createdAt: observed.createdAt,
  modifiedAt: observed.modifiedAt,
});