sailfork / t3code-github / .repos/ alchemy-effect/ packages/ integration-tests/ test/ Cloudflare/ K2/ ReadSubscription.test.ts

import * as Cloudflare from "@alchemy.run/cloudflare";
import { CloudflareEnvironment } from "@alchemy.run/cloudflare/CloudflareEnvironment";
import { makeK2LocalAuth } from "@alchemy.run/cloudflare/K2/K2Http";
import { makeReadSubscriptionClient } from "@alchemy.run/cloudflare/K2/ReadSubscriptionHttp";
import { expect } from "@alchemy.run/test";
import * as Test from "@alchemy.run/test/Alchemy";
import * as k2 from "@distilled.cloud/cloudflare/k2";
import { Action } from "alchemy/Action";
import { createHostRuntimeContext, ServerHost } from "alchemy/Runtime/Process";
import { RuntimeContext } from "alchemy/RuntimeContext";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import { MinimumLogLevel } from "effect/References";
import * as Schedule from "effect/Schedule";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";
import { waitForStreamGone } from "./k2-test-utils.ts";

const { test } = Test.make({ providers: Cloudflare.providers() });

const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info");

const Event = Schema.Struct({ source: Schema.String, index: Schema.Number });
type Event = typeof Event.Type;

class NotConsumed extends Data.TaggedError("NotConsumed")<{ distinct: number }> {}

const decoder = new TextDecoder();

/**
 * The `*Local` layers end to end, from an Action: `WriteStreamLocal` and
 * `StreamSinkLocal` produce typed records, and `consumeStreamRecords`
 * (`StreamEventSourcePolling` over `ReadSubscriptionLocal`, on an in-process
 * `ServerHost`) consumes them. The handler fails once to exercise
 * nack → redelivery. The ReadSubscription client is then driven directly
 * against a second subscription.
 */
test.provider(
  "Local producers, consumeStreamRecords polling, and the ReadSubscription client",
  (stack) =>
    Effect.gen(function* () {
      const { accountId } = yield* yield* CloudflareEnvironment;

      yield* stack.destroy();

      const out = yield* stack.deploy(
        Effect.gen(function* () {
          const events = yield* Cloudflare.K2.Stream("LocalEvents", { http: true, schema: Event });
          const poller = yield* Cloudflare.K2.Subscription("Poller", {
            stream: events,
            startAt: "earliest",
          });
          const reader = yield* Cloudflare.K2.Subscription("Reader", {
            stream: events,
            startAt: "earliest",
          });

          const Roundtrip = Action(
            "Roundtrip",
            Effect.gen(function* () {
              const writer = yield* Cloudflare.K2.WriteStream(events);
              const sink = yield* Cloudflare.K2.StreamSink(events);

              const seen = new Set<string>();
              let failures = 0;
              const host = createHostRuntimeContext("Test.K2Host")("K2Host");
              yield* Cloudflare.K2.consumeStreamRecords(
                events,
                {
                  subscription: poller,
                  workerId: "alchemy-k2-poller",
                  idleBackoff: "500 millis",
                  concurrency: 2,
                },
                (records) =>
                  Effect.gen(function* () {
                    if (failures === 0) {
                      failures++;
                      return yield* Effect.fail("first delivery fails" as const);
                    }
                    yield* records.pipe(
                      Stream.runForEach((record) =>
                        Effect.sync(() => seen.add(`${record.value.source}:${record.value.index}`)),
                      ),
                    );
                  }),
              ).pipe(
                Effect.provide(
                  Layer.provideMerge(
                    Cloudflare.K2.StreamEventSourcePolling,
                    Layer.mergeAll(
                      Cloudflare.K2.ReadSubscriptionLocal,
                      Layer.succeed(ServerHost, host),
                    ),
                  ),
                ),
              );

              return Effect.fn(function* () {
                yield* writer.send([0, 1, 2, 3, 4].map((index) => ({ source: "writer", index })));
                yield* Stream.range(0, 19).pipe(
                  Stream.map((index): Event => ({ source: "sink", index })),
                  Stream.run(sink),
                );

                const { program } = yield* host.exports;
                const drained = Effect.suspend(() =>
                  seen.size >= 25
                    ? Effect.succeed(seen.size)
                    : Effect.fail(new NotConsumed({ distinct: seen.size })),
                ).pipe(
                  Effect.retry({
                    schedule: Schedule.max([Schedule.spaced("1 second"), Schedule.recurs(60)]),
                  }),
                );
                const distinct = yield* Effect.raceFirst(
                  drained,
                  program.pipe(Effect.andThen(Effect.never)),
                );
                return { distinct, failures, seen: [...seen].sort() };
              });
            }).pipe(
              Effect.provide(
                Layer.mergeAll(Cloudflare.K2.WriteStreamLocal, Cloudflare.K2.StreamSinkLocal),
              ),
            ),
          );

          return {
            roundtrip: yield* Roundtrip({}),
            streamId: events.streamId,
            readerId: reader.subscriptionId,
          };
        }),
      );

      expect(out.roundtrip.distinct).toEqual(25);
      expect(out.roundtrip.failures).toEqual(1);
      expect(out.roundtrip.seen).toContain("writer:4");
      expect(out.roundtrip.seen).toContain("sink:19");

      // Drive the client directly (current credentials) on the reader.
      const auth = yield* makeK2LocalAuth;
      const client = makeReadSubscriptionClient(auth, {
        streamId: Effect.succeed(out.streamId),
        subscriptionId: Effect.succeed(out.readerId),
      });
      const consume = (workerId: string, maxRecords: number) =>
        client.consume({ workerId, maxRecords }).pipe(
          Effect.retry({
            while: (e) => e._tag === "K2BatchPending",
            schedule: Schedule.max([Schedule.spaced("500 millis"), Schedule.recurs(10)]),
          }),
        );

      const first = yield* consume("reader-a", 3);
      expect(Option.isSome(first)).toBe(true);
      const batch = Option.getOrThrow(first);
      expect(batch.records.length).toEqual(3);
      expect(JSON.parse(decoder.decode(batch.records[0]!.content))).toEqual({
        source: "writer",
        index: 0,
      });
      expect(batch.records[0]!.headers["content-type"]).toEqual("application/json");
      expect(batch.records[0]!.timestamp).toBeInstanceOf(Date);

      // The same worker gets the same batch back; extend refreshes the lease.
      const again = Option.getOrThrow(yield* consume("reader-a", 3));
      expect(again.batchId).toEqual(batch.batchId);
      const extended = yield* client.extend(batch);
      expect(extended.leasedUntil.getTime()).toBeGreaterThan(0);

      // nack → redelivered under a new batch id.
      yield* client.nack(batch);
      const redelivered = Option.getOrThrow(yield* consume("reader-a", 3));
      expect(redelivered.batchId).not.toEqual(batch.batchId);
      expect(redelivered.records.map((r) => decoder.decode(r.content))).toEqual(
        batch.records.map((r) => decoder.decode(r.content)),
      );

      // ack advances the subscription; acking twice succeeds; extending an
      // acknowledged batch is K2LeaseLost.
      yield* client.ack(redelivered);
      yield* client.ack(redelivered);
      const lost = yield* client.extend(redelivered).pipe(Effect.flip);
      expect(lost._tag).toEqual("K2LeaseLost");

      // Drain the rest; an empty subscription consumes None.
      const rest = Option.getOrThrow(yield* consume("reader-a", 1000));
      expect(rest.records.length).toEqual(22);
      yield* client.ack(rest);
      const empty = yield* consume("reader-a", 10);
      expect(Option.isNone(empty)).toBe(true);

      yield* stack.destroy();
      yield* waitForStreamGone(accountId, out.streamId);
    }).pipe(logLevel),
  { tags: ["provider:cloudflare", "provider:cloudflare:k2", "live"], timeout: 120_000 },
);

// Deploy-time half of `consumeStreamRecords` without `subscription`: it
// declares a Subscription owned by the consuming host.
test.provider(
  "consumeStreamRecords creates a host-owned subscription",
  (stack) =>
    Effect.gen(function* () {
      const { accountId } = yield* yield* CloudflareEnvironment;

      yield* stack.destroy();

      const host = createHostRuntimeContext("Test.K2Host")("K2Host");
      const out = yield* stack.deploy(
        Effect.gen(function* () {
          const events = yield* Cloudflare.K2.Stream("AutoEvents");
          yield* Cloudflare.K2.consumeStreamRecords(events, { startAt: "earliest" }, (records) =>
            Stream.runDrain(records),
          ).pipe(
            Effect.provide(
              Layer.provideMerge(
                Cloudflare.K2.StreamEventSourcePolling,
                Layer.mergeAll(
                  Cloudflare.K2.ReadSubscriptionLocal,
                  Layer.succeed(ServerHost, host),
                  // A real host provides its runtime context; outputs the
                  // client reads are bound into the host's env.
                  Layer.succeed(RuntimeContext, host),
                ),
              ),
            ),
          );
          return { streamId: events.streamId };
        }),
      );

      const subscriptions = yield* k2.listSubscriptions({ streamId: out.streamId });
      expect(subscriptions.length).toEqual(1);
      expect(subscriptions[0]?.startAt.type).toEqual("earliest");

      yield* stack.destroy();
      yield* waitForStreamGone(accountId, out.streamId);
    }).pipe(logLevel),
  { tags: ["provider:cloudflare", "provider:cloudflare:k2", "live"], timeout: 120_000 },
);