sailfork / t3code-github / .repos/ alchemy-effect/ packages/ integration-tests/ test/ Cloudflare/ Basin/ Pipeline.test.ts

import * as Cloudflare from "@alchemy.run/cloudflare";
import { CloudflareEnvironment } from "@alchemy.run/cloudflare/CloudflareEnvironment";
import { expect } from "@alchemy.run/test";
import * as Test from "@alchemy.run/test/Alchemy";
import * as pipelines from "@distilled.cloud/cloudflare/pipelines";
import * as Output from "alchemy/Output";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import { MinimumLogLevel } from "effect/References";
import { apiToken, PageView } from "./fixtures/pipelines-shared.ts";

const { test } = Test.make({ providers: Cloudflare.providers() });

const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info");

const getPipeline = (accountId: string, pipelineId: string) =>
  pipelines.getV1Pipeline({ accountId, pipelineId });

/**
 * The whole Basin topology in one deploy: a typed stream, a catalog sink
 * that takes the Catalog resource (so it is created after the catalog),
 * and the SQL pipeline between them.
 */
const lakehouse = (sql: (stream: string, sink: string) => string) =>
  Effect.gen(function* () {
    const token = yield* apiToken;
    const bucket = yield* Cloudflare.R2.Bucket("LakeBucket", { forceDestroy: true });
    const catalog = yield* Cloudflare.Basin.Catalog("Catalog", {
      bucket,
    });
    const stream = yield* Cloudflare.Basin.Stream("PageViews", { schema: PageView });
    const sink = yield* Cloudflare.Basin.Sink("PageViewTable", {
      type: "basin_catalog",
      catalog,
      table: { namespace: "web", name: "page_views" },
      token,
    });
    const pipeline = yield* Cloudflare.Basin.Pipeline("Etl", {
      sql: Output.all(stream.name, sink.name).pipe(
        Output.map(([streamName, sinkName]) => sql(streamName, sinkName)),
      ),
    });
    return { stream, sink, pipeline };
  });

const passthrough = (stream: string, sink: string) => `INSERT INTO ${sink} SELECT * FROM ${stream}`;

test.provider(
  "SQL is validated during plan; whitespace-only changes are a no-op",
  (stack) =>
    Effect.gen(function* () {
      const { accountId } = yield* yield* CloudflareEnvironment;
      yield* stack.destroy();

      const deployed = yield* stack.deploy(lakehouse(passthrough));
      expect(deployed.sink.type).toEqual("basin_catalog");
      expect(deployed.pipeline.pipelineId).toBeTruthy();

      // Reformatted SQL is the same statement.
      const reformatted = yield* stack.plan(
        lakehouse((stream, sink) => `  INSERT INTO ${sink}\n    SELECT *\n    FROM ${stream};  `),
      );
      expect(reformatted.resources.Etl!.action).toEqual("noop");

      // Invalid SQL fails the plan — before the running pipeline is deleted.
      const invalid = yield* stack
        .plan(lakehouse((stream, sink) => `INSERT INTO ${sink} SELEC * FROM ${stream}`))
        .pipe(Effect.exit);
      expect(Exit.isFailure(invalid)).toBe(true);
      expect(String(Exit.isFailure(invalid) ? invalid.cause : "")).toContain("invalid SQL");

      // A valid change is a replacement.
      const filtered = yield* stack.plan(
        lakehouse((stream, sink) => `INSERT INTO ${sink} SELECT * FROM ${stream} WHERE url != ''`),
      );
      expect(filtered.resources.Etl!.action).toEqual("replace");

      const live = yield* getPipeline(accountId, deployed.pipeline.pipelineId);
      expect(live.id).toEqual(deployed.pipeline.pipelineId);

      yield* stack.destroy();
    }).pipe(logLevel),
  {
    tags: [
      "provider:cloudflare",
      "provider:cloudflare:pipelines",
      "provider:cloudflare:r2",
      "live",
    ],
    timeout: 240_000,
  },
);