sailfork / t3code-github / .repos/ alchemy-effect/ packages/ integration-tests/ test/ Cloudflare/ Basin/ Sink.test.ts

import * as Cloudflare from "@alchemy.run/cloudflare";
import { CloudflareEnvironment } from "@alchemy.run/cloudflare/CloudflareEnvironment";
import { expect } from "@alchemy.run/test";
import * as Test from "@alchemy.run/test/Alchemy";
import * as pipelines from "@distilled.cloud/cloudflare/pipelines";
import * as Effect from "effect/Effect";
import * as Redacted from "effect/Redacted";
import { MinimumLogLevel } from "effect/References";
import * as Schedule from "effect/Schedule";
import { apiToken, r2Credentials } from "./fixtures/pipelines-shared.ts";

const { test } = Test.make({ providers: Cloudflare.providers() });

const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info");

const getSink = (accountId: string, sinkId: string) => pipelines.getSink({ accountId, sinkId });

const expectSinkGone = (accountId: string, sinkId: string) =>
  getSink(accountId, sinkId).pipe(
    Effect.flatMap(() => Effect.fail({ _tag: "SinkNotDeleted" } as const)),
    Effect.catchTag("SinkNotFound", () => Effect.void),
    Effect.retry({
      while: (e) => e._tag === "SinkNotDeleted",
      schedule: Schedule.max([Schedule.exponential("500 millis"), Schedule.recurs(10)]),
    }),
  );

const tags = [
  "provider:cloudflare",
  "provider:cloudflare:pipelines",
  "provider:cloudflare:r2",
  "live",
];

const rotated = {
  accessKeyId: Redacted.make("rotated-access-key-id"),
  secretAccessKey: Redacted.make("rotated-secret-access-key"),
};

test.provider(
  "r2 sink: credential rotation and equivalent defaults never replace",
  (stack) =>
    Effect.gen(function* () {
      const { accountId } = yield* yield* CloudflareEnvironment;
      const credentials = yield* r2Credentials;
      yield* stack.destroy();

      const sink = (opts: {
        credentials: typeof credentials;
        explicitDefaults?: boolean;
        path?: string;
      }) =>
        Effect.gen(function* () {
          const bucket = yield* Cloudflare.R2.Bucket("SinkBucket", { forceDestroy: true });
          return yield* Cloudflare.Basin.Sink("Sink", {
            type: "r2",
            format: opts.explicitDefaults ? { type: "json" } : undefined,
            config: {
              bucket: bucket.bucketName,
              credentials: opts.credentials,
              path: opts.path,
              rollingPolicy: opts.explicitDefaults ? { intervalSeconds: 300 } : undefined,
            },
          });
        });

      const initial = yield* stack.deploy(sink({ credentials }));

      // Rotated credentials are write-only and cannot be updated: no replace.
      const rotatePlan = yield* stack.plan(sink({ credentials: rotated }));
      expect(rotatePlan.resources.Sink!.action).toEqual("noop");

      // Spelling out the defaults is equivalent.
      const defaultsPlan = yield* stack.plan(sink({ credentials, explicitDefaults: true }));
      expect(defaultsPlan.resources.Sink!.action).toEqual("noop");

      // A destination change still replaces.
      const pathPlan = yield* stack.plan(sink({ credentials, path: "moved" }));
      expect(pathPlan.resources.Sink!.action).toEqual("replace");

      const after = yield* stack.deploy(sink({ credentials: rotated }));
      expect(after.sinkId).toEqual(initial.sinkId);

      yield* stack.destroy();
      yield* expectSinkGone(accountId, initial.sinkId);
    }).pipe(logLevel),
  { tags, timeout: 240_000 },
);

test.provider(
  "catalog sink: config form ⇄ catalog form and r2_data_catalog ⇄ basin_catalog are no-ops",
  (stack) =>
    Effect.gen(function* () {
      const { accountId } = yield* yield* CloudflareEnvironment;
      const token = yield* apiToken;
      yield* stack.destroy();

      const lakehouse = Effect.gen(function* () {
        const bucket = yield* Cloudflare.R2.Bucket("LakeBucket", { forceDestroy: true });
        const catalog = yield* Cloudflare.Basin.Catalog("Catalog", {
          bucket,
        });
        return { bucket, catalog };
      });

      // The catalog's attributes are all stable, so the old bucket-name
      // spelling does not wait for it — enable it in a first deploy.
      yield* stack.deploy(lakehouse);

      const initial = yield* stack.deploy(
        Effect.gen(function* () {
          // The old spelling.
          const { catalog } = yield* lakehouse;
          return yield* Cloudflare.Pipelines.Sink("Table", {
            type: "r2_data_catalog",
            config: {
              bucket: catalog.bucketName,
              namespace: "web",
              tableName: "page_views",
              token,
            },
          });
        }),
      );
      expect(initial.type).toEqual("r2_data_catalog");
      expect(initial.tableName).toEqual("page_views");
      const live = yield* getSink(accountId, initial.sinkId);
      expect(live.config?.bucket).toEqual(initial.bucket);

      const catalogForm = (sinkToken: Redacted.Redacted<string>) =>
        Effect.gen(function* () {
          const { catalog } = yield* lakehouse;
          return yield* Cloudflare.Basin.Sink("Table", {
            type: "basin_catalog",
            catalog,
            table: { namespace: "web", name: "page_views" },
            token: sinkToken,
          });
        });

      const plan = yield* stack.plan(catalogForm(token));
      expect(plan.resources.Table!.action).toEqual("noop");

      // A rotated token never replaces (the table already exists, so a
      // catalog sink could not be recreated on it anyway).
      const rotatePlan = yield* stack.plan(catalogForm(Redacted.make("rotated-token")));
      expect(rotatePlan.resources.Table!.action).toEqual("noop");

      const switched = yield* stack.deploy(catalogForm(token));
      expect(switched.sinkId).toEqual(initial.sinkId);

      yield* stack.destroy();
      yield* expectSinkGone(accountId, initial.sinkId);
    }).pipe(logLevel),
  { tags, timeout: 240_000 },
);