sailfork / t3code-github / .repos/ alchemy-effect/ packages/ cloudflare/ src/ K2/ StreamEventSourcePoller.ts

import * as Binding from "alchemy/Binding";
import * as Namespace from "alchemy/Namespace";
import { ServerHost } from "alchemy/Runtime/Process";
import type { RuntimeContext } from "alchemy/RuntimeContext";
import * as Cause from "effect/Cause";
import type * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { decodeWithSchema } from "./K2Codec.ts";
import type { Batch, ConsumedRecord, K2SchemaError } from "./K2Types.ts";
import { ReadSubscription, type ReadSubscriptionClient } from "./ReadSubscription.ts";
import type { Stream as K2Stream } from "./Stream.ts";
import {
  StreamEventSource,
  type ConsumeStreamRecordsProps,
  type ConsumeStreamRecordsHandler,
  type StreamEventSourceService,
} from "./StreamEventSource.ts";
import { Subscription } from "./Subscription.ts";

/**
 * Polling implementation of {@link StreamEventSource} for hosts with a
 * long-lived process (any host providing `ServerHost`: Fly, ECS, EC2,
 * Railway, Hetzner, Docker). Requires a `ReadSubscription` implementation.
 *
 * :::note
 * TODO: Workers cannot consume K2 yet — K2 has no push consumers (they are
 * on Cloudflare's roadmap), so no Worker layer ships.
 * :::
 * ### Providing the Layer
 * **Example:** Poll with a scoped token
 * ```typescript
 * Effect.gen(function* () {
 *   yield* Cloudflare.K2.consumeStreamRecords(Orders, handler);
 * }).pipe(
 *   Effect.provide(
 *     Layer.provideMerge(
 *       Cloudflare.K2.StreamEventSourcePolling,
 *       Cloudflare.K2.ReadSubscriptionHttp,
 *     ),
 *   ),
 * );
 * ```
 *
 * @layer
 * @provides Cloudflare.K2.StreamEventSource
 * @product K2
 * @category Storage & Databases
 */
export const StreamEventSourcePolling = Layer.effect(
  StreamEventSource,
  Effect.gen(function* () {
    const { run } = yield* ServerHost;
    const readSubscription = yield* ReadSubscription;

    return Effect.fn(function* (
      stream: K2Stream<any>,
      props: ConsumeStreamRecordsProps,
      process: ConsumeStreamRecordsHandler<any, any>,
    ) {
      const host = yield* Binding.Host;
      const hostId = host?.LogicalId ?? "Host";
      // Declared in both phases: at runtime the declaration resolves to the
      // deployed subscription's outputs, which the client reads.
      const subscription =
        props.subscription ??
        (yield* Namespace.push(
          hostId,
          Subscription(`${stream.LogicalId}Subscription`, {
            stream,
            startAt: props.startAt,
          }),
        ));
      const client = yield* readSubscription(subscription);
      const decode = stream.RecordSchema
        ? decodeWithSchema(stream.RecordSchema)
        : (records: ReadonlyArray<ConsumedRecord>) => Effect.succeed(records);

      yield* run(
        Effect.gen(function* () {
          const workerId =
            props.workerId ??
            `${hostId}-${(yield* Effect.sync(() => crypto.randomUUID())).slice(0, 8)}`;
          yield* pollSubscription({
            client,
            workerId,
            maxRecords: props.maxRecords,
            idleBackoff: props.idleBackoff,
            concurrency: props.concurrency,
            decode,
            process,
          });
        }),
      );
    }) as StreamEventSourceService;
  }),
);

// ---- the pull loop -----------------------------------------------------------

/** `consume` errors that mean "try again", not "give up". */
const RETRYABLE_CONSUME_ERRORS: ReadonlyArray<string> = [
  "K2Unavailable",
  "K2ReadFailed",
  "K2LeasesExhausted",
  "K2BatchPending",
];

/** Bounded backoff for retryable `consume` errors: ~30 s in total. */
const consumeRetry = Schedule.max([
  Schedule.min([Schedule.exponential("250 millis"), Schedule.spaced("5 seconds")]),
  Schedule.recurs(8),
]);

interface PollOptions<A, Req> {
  client: ReadSubscriptionClient;
  /** Worker id holding the leases. Must be unique per concurrent poller. */
  workerId: string;
  /** @default 100 */
  maxRecords?: number;
  /** Sleep after an empty batch or a failed iteration. @default "1 second" */
  idleBackoff?: Duration.Input;
  /** Extend the five-minute lease this often while the handler runs. @default "2 minutes" */
  extendEvery?: Duration.Input;
  /** Turn consumed records into the handler's elements (e.g. schema decode). */
  decode: (
    records: ReadonlyArray<ConsumedRecord>,
  ) => Effect.Effect<ReadonlyArray<A>, K2SchemaError>;
  process: (records: Stream.Stream<A>) => Effect.Effect<void, unknown, Req>;
}

/**
 * Keep a batch's lease alive while its handler runs. Stops on `K2LeaseLost`
 * (the batch was acked, nacked, or redelivered elsewhere).
 */
const keepLeaseAlive = (client: ReadSubscriptionClient, batch: Batch, every: Duration.Input) =>
  Effect.sleep(every).pipe(
    Effect.andThen(client.extend(batch)),
    Effect.forever,
    Effect.catchTag("K2LeaseLost", () =>
      Effect.logWarning(`K2 lease lost for batch ${batch.batchId}; it may be redelivered`),
    ),
    Effect.catchCause((cause) =>
      Effect.logWarning(`K2 lease extension failed for batch ${batch.batchId}`, cause),
    ),
  );

/**
 * One poll: lease a batch, run the handler, then ack it on success or nack
 * it on failure. Returns whether a batch was processed.
 */
const pollOnce = <A, Req>(
  options: PollOptions<A, Req>,
): Effect.Effect<boolean, unknown, Req | RuntimeContext> =>
  Effect.gen(function* () {
    const { client } = options;
    const leased = yield* client
      .consume({ workerId: options.workerId, maxRecords: options.maxRecords })
      .pipe(
        Effect.retry({
          while: (e) => RETRYABLE_CONSUME_ERRORS.includes(e._tag),
          schedule: consumeRetry,
        }),
      );
    if (Option.isNone(leased)) return false;
    const batch = leased.value;

    const outcome = yield* Effect.scoped(
      Effect.gen(function* () {
        yield* Effect.forkScoped(keepLeaseAlive(client, batch, options.extendEvery ?? "2 minutes"));
        return yield* Effect.exit(
          options
            .decode(batch.records)
            .pipe(Effect.flatMap((values) => options.process(Stream.fromIterable(values)))),
        );
      }),
    );

    if (Exit.isSuccess(outcome)) {
      yield* client.ack(batch);
    } else {
      yield* Effect.logWarning(
        `K2 handler failed for batch ${batch.batchId}; releasing it for redelivery`,
        Cause.pretty(outcome.cause),
      );
      yield* client.nack(batch);
    }
    return true;
  });

/**
 * Poll forever: an empty batch or a failed iteration sleeps `idleBackoff`;
 * a processed batch polls again immediately. Never fails.
 */
const pollForever = <A, Req>(
  options: PollOptions<A, Req>,
): Effect.Effect<never, never, Req | RuntimeContext> =>
  pollOnce(options).pipe(
    Effect.catchCause((cause) =>
      Effect.logWarning(`K2 poll failed for worker ${options.workerId}`, cause).pipe(
        Effect.as(false),
      ),
    ),
    Effect.flatMap((processed) =>
      processed ? Effect.void : Effect.sleep(options.idleBackoff ?? "1 second"),
    ),
    Effect.forever,
  );

/** Run `concurrency` pollers, each with its own worker id (`${workerId}-${i}`). */
const pollSubscription = <A, Req>(
  options: PollOptions<A, Req> & { concurrency?: number },
): Effect.Effect<void, never, Req | RuntimeContext> =>
  Effect.forEach(
    Array.from({ length: Math.max(1, options.concurrency ?? 1) }, (_, i) => i),
    (i) => pollForever({ ...options, workerId: `${options.workerId}-${i}` }),
    { concurrency: "unbounded", discard: true },
  );