sailfork / t3code-github / .repos/ alchemy-effect/ packages/ integration-tests/ test/ Cloudflare/ Basin/ WriteStream.test.ts

import * as Cloudflare from "@alchemy.run/cloudflare";
import { CloudflareEnvironment } from "@alchemy.run/cloudflare/CloudflareEnvironment";
import { expect } from "@alchemy.run/test";
import * as Test from "@alchemy.run/test/Alchemy";
import * as pipelines from "@distilled.cloud/cloudflare/pipelines";
import { Action } from "alchemy/Action";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as HttpClient from "effect/http/HttpClient";
import { MinimumLogLevel } from "effect/References";
import * as Schedule from "effect/Schedule";
import * as EffectStream from "effect/Stream";
import PipelinesHttpWorker from "./fixtures/pipelines-http-worker.ts";
import { PageView, PageViews } from "./fixtures/pipelines-stream.ts";
import PipelinesWorker from "./fixtures/pipelines-worker.ts";

const { test } = Test.make({ providers: Cloudflare.providers() });

const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info");

class WorkerNotReady extends Data.TaggedError("WorkerNotReady")<{
  status: number;
  body: string;
}> {}

// A fresh workers.dev URL takes a few seconds to serve; retry until 200.
const getJson = (url: string) =>
  HttpClient.get(url).pipe(
    Effect.flatMap((res) =>
      res.status === 200
        ? res.json
        : res.text.pipe(
            Effect.flatMap((body) => Effect.fail(new WorkerNotReady({ status: res.status, body }))),
          ),
    ),
    Effect.retry({
      while: (e): e is WorkerNotReady => e instanceof WorkerNotReady,
      schedule: Schedule.max([
        Schedule.min([Schedule.exponential("500 millis"), Schedule.spaced("5 seconds")]),
        Schedule.recurs(20),
      ]),
    }),
  );

// A just-created stream's ingest host can take a moment to accept writes.
const ingestRetry = {
  while: (e: { _tag: string; reason?: string }) =>
    e._tag === "StreamSendError" && e.reason === "SendFailed",
  schedule: Schedule.exponential("1 second"),
  times: 6,
} as const;

test.provider(
  "typed WriteStream and StreamSink: native binding (Worker) and HTTP ingest (Action)",
  (stack) =>
    Effect.gen(function* () {
      const { accountId } = yield* yield* CloudflareEnvironment;
      yield* stack.destroy();

      const deployed = yield* stack.deploy(
        Effect.gen(function* () {
          const stream = yield* PageViews;
          const worker = yield* PipelinesWorker;

          // WriteStreamLocal / StreamSinkLocal POST to the stream's HTTP
          // ingest endpoint (`http: true`) with the current credentials.
          const Seed = Action(
            "Seed",
            Effect.gen(function* () {
              const views = yield* Cloudflare.Basin.WriteStream(stream);
              const sink = yield* Cloudflare.Basin.StreamSink(stream);
              return Effect.fn(function* () {
                yield* views
                  .send([new PageView({ url: "/seed", at: new Date(), tags: ["seed"] })])
                  .pipe(Effect.retry(ingestRetry));
                yield* EffectStream.range(1, 50).pipe(
                  EffectStream.map(
                    (i) => new PageView({ url: `/seed/${i}`, at: new Date(), tags: [] }),
                  ),
                  EffectStream.run(sink),
                  Effect.retry(ingestRetry),
                );
                const invalid = yield* views
                  .send([{ url: "/x", at: "nope", tags: [] } as unknown as PageView])
                  .pipe(
                    Effect.as("none"),
                    Effect.catchTag("StreamSendError", (e) =>
                      Effect.succeed(e.reason ?? "unknown"),
                    ),
                  );
                return { invalid };
              });
            }).pipe(
              Effect.provide(Cloudflare.Basin.WriteStreamLocal),
              Effect.provide(Cloudflare.Basin.StreamSinkLocal),
            ),
          );
          const seeded = yield* Seed({});

          return {
            streamId: stream.streamId,
            url: worker.url.as<string>(),
            seeded,
          };
        }),
      );

      expect(deployed.seeded).toEqual({ invalid: "InvalidRecord" });

      const live = yield* pipelines.getStream({ accountId, streamId: deployed.streamId });
      expect(live.http.enabled).toBe(true);
      expect(live.http.authentication).toBe(true);

      expect(yield* getJson(`${deployed.url}/send`)).toEqual({ sent: 2 });
      expect(yield* getJson(`${deployed.url}/sink?count=300`)).toEqual({ sent: 300 });
      expect(yield* getJson(`${deployed.url}/invalid`)).toEqual({
        reason: "InvalidRecord",
        index: 0,
      });

      yield* stack.destroy();
    }).pipe(logLevel),
  {
    tags: [
      "provider:cloudflare",
      "provider:cloudflare:pipelines",
      "provider:cloudflare:worker",
      "live",
    ],
    timeout: 240_000,
  },
);

// WriteStreamHttp/StreamSinkHttp mint a scoped `Pipelines Send` API token.
test.provider(
  "WriteStreamHttp and StreamSinkHttp send through a scoped token",
  (stack) =>
    Effect.gen(function* () {
      yield* stack.destroy();
      const { url } = yield* stack.deploy(
        Effect.gen(function* () {
          yield* PageViews;
          const worker = yield* PipelinesHttpWorker;
          return { url: worker.url.as<string>() };
        }),
      );
      expect(yield* getJson(`${url}/send`)).toEqual({ sent: 21 });
      yield* stack.destroy();
    }).pipe(logLevel),
  {
    tags: [
      "provider:cloudflare",
      "provider:cloudflare:pipelines",
      "provider:cloudflare:worker",
      "live",
    ],
    timeout: 240_000,
  },
);