sailfork / t3code-github / .repos/ alchemy-effect/ packages/ cloudflare/ src/ Queues/ EventSource.ts

import type * as cf from "@cloudflare/workers-types";
import * as Namespace from "alchemy/Namespace";
import type { FunctionContext } from "alchemy/Runtime/Function";
import { RuntimeContext } from "alchemy/RuntimeContext";
import * as DurationUtil from "alchemy/Util/Duration";
import * as Cause from "effect/Cause";
import * as Context from "effect/Context";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Stream from "effect/Stream";
import { isWorkerEvent, Worker } from "../Workers/Worker.ts";
import { Consumer } from "./Consumer.ts";
import type { Queue } from "./Queue.ts";

/**
 * Subscriber settings — the same shape Cloudflare's `Consumer`
 * accepts. `consumeQueueMessages(queue, props, handler)` passes these
 * through to the auto-created `Cloudflare.Queues.Consumer` so a single
 * call captures both runtime and deploy-time intent.
 */
export interface MessagesProps {
  /** Maximum messages per batch. */
  batchSize?: number;
  /** Maximum concurrent invocations. */
  maxConcurrency?: number;
  /** Maximum delivery attempts before dead-lettering. */
  maxRetries?: number;
  /**
   * Wait time before flushing a partial batch. Rounded up to whole
   * milliseconds when forwarded to Cloudflare.
   */
  maxWaitTime?: Duration.Input;
  /**
   * Backoff applied to a retry. Rounded up to whole seconds when
   * forwarded to Cloudflare.
   */
  retryDelay?: Duration.Input;
  /**
   * Queue that receives messages after `maxRetries` failed deliveries.
   * Pass the Queue resource, a `Queue.ref(...)` to one, or a queue name.
   *
   * Cloudflare does not drain or alert on a dead-letter queue. A message
   * left there expires after the queue's retention period, so call
   * `consumeQueueMessages` on the dead-letter queue as well to log,
   * persist, or alert on what arrives.
   */
  deadLetterQueue?: string | Queue;
}

/**
 * Convert a {@link MessagesProps} (with `Duration.Input` time fields)
 * into the numeric settings shape Cloudflare's `Consumer` API
 * expects. `maxWaitTime` is rounded up to whole milliseconds and
 * `retryDelay` to whole seconds.
 *
 * Exposed for testing and for callers that want to mirror the
 * conversion when wiring `Consumer` directly.
 */
export const toConsumerSettings = (props: MessagesProps) => ({
  batchSize: props.batchSize,
  maxConcurrency: props.maxConcurrency,
  maxRetries: props.maxRetries,
  maxWaitTimeMs: DurationUtil.toMillis(props.maxWaitTime),
  retryDelay: DurationUtil.toSeconds(props.retryDelay),
});

/**
 * A single queue message handed to the subscribe handler. Mirrors
 * Cloudflare's runtime `Message<Body>` shape so per-message
 * `ack()` / `retry()` semantics match the platform docs.
 */
export type Message<Body = unknown> = cf.Message<Body>;

/**
 * Subscribe to a Cloudflare Queue with an Effect stream handler.
 *
 * Mirrors `AWS.SQS.consumeQueueMessages(queue, handler)` on the
 * Cloudflare side. Wires both halves of the consumer in one call:
 *
 * - **Runtime**: registers a `queue` event listener on the Worker.
 *   Each batch is piped through `process` as a `Stream.Stream`.
 * - **Deploy-time**: yields a `Cloudflare.Queues.Consumer` resource
 *   so Cloudflare actually dispatches messages from `queue` to
 *   this Worker. No manual `Consumer` wiring needed in
 *   `alchemy.run.ts`.
 *
 * Acking semantics: if `process` succeeds, every message in the
 * batch is `ack()`ed; if it fails, every message is `retry()`ed
 * and Cloudflare applies `maxRetries` / `retryDelay` from the
 * settings before dead-lettering. Per-message control is still
 * available by calling `msg.ack()` / `msg.retry()` inside the
 * handler.
 *
 * ### Consuming a Queue
 * **Example:** Handle each batch as a Stream
 * ```typescript
 * yield* Cloudflare.Queues.consumeQueueMessages<MyEvent>(queueResource, (stream) =>
 *   Stream.runForEach(stream, (msg) => Effect.log(`event ${msg.body.id}`)),
 * );
 * ```
 *
 * ### Tuning Batching and Retries
 * **Example:** Batch size, wait time, and retry backoff
 * ```typescript
 * yield* Cloudflare.Queues.consumeQueueMessages<MyEvent>(
 *   queueResource,
 *   {
 *     batchSize: 25,
 *     maxRetries: 3,
 *     maxWaitTime: "5 seconds",
 *     retryDelay: Duration.seconds(30),
 *   },
 *   (stream) => Stream.runForEach(stream, (msg) => Effect.log(`event ${msg.body.id}`)),
 * );
 * ```
 *
 * ### Dead-Lettering Failed Messages
 * A message that fails `maxRetries` times moves to `deadLetterQueue`.
 * Cloudflare does not drain or alert on that queue, so consume it too.
 * Otherwise the message expires there unseen.
 *
 * **Example:** Dead-letter to a Queue resource and consume it
 * ```typescript
 * const deadLetters = yield* Cloudflare.Queues.Queue("DeadLetters");
 *
 * yield* Cloudflare.Queues.consumeQueueMessages<MyEvent>(
 *   queueResource,
 *   { maxRetries: 3, deadLetterQueue: deadLetters },
 *   (stream) => Stream.runForEach(stream, (msg) => Effect.log(`event ${msg.body.id}`)),
 * );
 *
 * yield* Cloudflare.Queues.consumeQueueMessages<MyEvent>(deadLetters, (stream) =>
 *   Stream.runForEach(stream, (msg) =>
 *     Effect.logError(`dead-lettered ${msg.body.id} after ${msg.attempts} attempts`),
 *   ),
 * );
 * ```
 *
 * **Example:** Dead-letter to a queue in another stack
 * ```typescript
 * // Deploy the "shared" stack first; its Worker consumes the dead-letter queue.
 * yield* Cloudflare.Queues.consumeQueueMessages<MyEvent>(
 *   queueResource,
 *   {
 *     maxRetries: 3,
 *     deadLetterQueue: yield* Cloudflare.Queues.Queue.ref("DeadLetters", {
 *       stack: "shared",
 *       stage: "production",
 *     }),
 *   },
 *   (stream) => Stream.runForEach(stream, (msg) => Effect.log(`event ${msg.body.id}`)),
 * );
 * ```
 *
 * **Example:** Dead-letter to a queue by name
 * ```typescript
 * // A queue Alchemy doesn't manage; something else must consume it.
 * yield* Cloudflare.Queues.consumeQueueMessages<MyEvent>(
 *   queueResource,
 *   { maxRetries: 3, deadLetterQueue: "orders-dead-letters" },
 *   (stream) => Stream.runForEach(stream, (msg) => Effect.log(`event ${msg.body.id}`)),
 * );
 * ```
 *
 * @binding
 * @product Queues
 * @category Storage & Databases
 */
export function consumeQueueMessages<Body = unknown>(
  queue: Queue,
  process: (stream: Stream.Stream<Message<Body>>) => Effect.Effect<void, unknown, any>,
): Effect.Effect<void, never, EventSource>;
export function consumeQueueMessages<Body = unknown>(
  queue: Queue,
  props: MessagesProps,
  process: (stream: Stream.Stream<Message<Body>>) => Effect.Effect<void, unknown, any>,
): Effect.Effect<void, never, EventSource>;
export function consumeQueueMessages<Body = unknown>(
  queue: Queue,
  propsOrProcess:
    | MessagesProps
    | ((stream: Stream.Stream<Message<Body>>) => Effect.Effect<void, unknown, any>),
  maybeProcess?: (stream: Stream.Stream<Message<Body>>) => Effect.Effect<void, unknown, any>,
): Effect.Effect<void, never, EventSource> {
  const [props, process] =
    typeof propsOrProcess === "function"
      ? [{} as MessagesProps, propsOrProcess]
      : [propsOrProcess, maybeProcess!];
  return EventSource.use((source) => source<Body>(queue, props, process));
}

// `Req` is the handler's requirements. The service registers the
// handler with the Worker's runtime context, where the runtime
// machinery provides bindings and `WorkerEnvironment` when the
// dispatch fires — so the requirement is satisfied at handler
// invocation, NOT at subscribe time. We drop `Req` from the return
// to keep init effects clean (mirrors `AWS.SQS.EventSourceService`).
export type EventSourceService = <Body = unknown>(
  queue: Queue,
  props: MessagesProps,
  process: (stream: Stream.Stream<Message<Body>>) => Effect.Effect<void, unknown, any>,
) => Effect.Effect<void, never, never>;

/**
 * Service tag for the Cloudflare Queue event source. Provided by
 * {@link EventSourceLive} on the Worker's runtime layer.
 */
export class EventSource extends Context.Service<EventSource, EventSourceService>()(
  "Cloudflare.Queues.EventSource",
) {}

/**
 * Runtime layer for {@link consumeQueueMessages}. Wires each
 * `consumeQueueMessages(queue, handler)` call in the Worker init phase to
 * a `queue` event listener on the runtime context, and (at deploy
 * time) yields the matching `Cloudflare.Queues.Consumer` resource so
 * Cloudflare dispatches messages from the queue to this Worker.
 *
 * Provide alongside other Cloudflare runtime layers (e.g.
 * `WriteQueueBinding`) on the Worker effect.
 */
export const EventSourceLive = Layer.effect(
  EventSource,
  Effect.gen(function* () {
    const host = yield* Worker;
    return Effect.fn(function* <Body, Req>(
      queue: Queue,
      props: MessagesProps,
      process: (stream: Stream.Stream<Message<Body>>) => Effect.Effect<void, unknown, any>,
    ) {
      // Deploy-time: yield the Consumer resource as a sibling of the
      // Worker so Cloudflare dispatches messages from the queue to it.
      // Skipped once running inside the deployed Worker (the global
      // guard), where the only work is registering the runtime handler
      // below. Namespaced under the host so the Consumer's logical
      // identity matches the previous Binding.Policy.
      if (!globalThis.__ALCHEMY_RUNTIME__) {
        yield* Namespace.push(
          host.LogicalId,
          // The engine creates / updates / destroys the Consumer
          // alongside the Worker's lifecycle; the consumer's
          // reconciler waits for the Worker upload to expose the
          // `queue` handler before completing (see PR #257 for the
          // 11001 retry).
          Consumer(`${queue.LogicalId}Consumer`, {
            queueId: queue.queueId,
            scriptName: host.workerName,
            settings: toConsumerSettings(props),
            deadLetterQueue:
              typeof props.deadLetterQueue === "string"
                ? props.deadLetterQueue
                : props.deadLetterQueue?.queueName,
          }),
        );
      }

      // Resolve the runtime context per-call rather than at layer
      // construction. Capturing it on the layer would leak the
      // requirement past `PlatformServices` exclusion when the
      // Worker typechecks its init effect.
      const ctx = (yield* RuntimeContext) as unknown as FunctionContext;
      // Capture the queue-name accessor once; the listener body
      // re-resolves it per event via `yield* QueueName`. A worker
      // can consume multiple queues — each subscribe registers its
      // own listener and they all see every queue event, so the
      // queue-name match is what scopes the handler.
      const QueueName = yield* queue.queueName;

      yield* ctx.listen<void, Req>((event) => {
        if (!isWorkerEvent(event) || event.type !== "queue") return;
        const batch = event.input as cf.MessageBatch<Body>;

        return Effect.gen(function* () {
          const queueName = yield* QueueName;
          if (batch.queue !== queueName) return;

          yield* process(Stream.fromIterable(batch.messages)).pipe(
            Effect.tap(() =>
              Effect.sync(() => {
                for (const msg of batch.messages) msg.ack();
              }),
            ),
            Effect.onError((cause) =>
              Effect.sync(() => {
                // Surface the failure so the operator sees what
                // tripped the retry path; without this the only
                // signal is the message reappearing on the next
                // attempt.
                console.error(
                  `[EventSource] handler failed on queue ` +
                    `"${queueName}": ${Cause.pretty(cause)}`,
                );
                for (const msg of batch.messages) msg.retry();
              }),
            ),
            Effect.ignoreCause,
          );
        });
      });
    }) as EventSourceService;
  }),
);