sailfork / t3code-github / .repos/ alchemy-effect/ packages/ integration-tests/ test/ Cloudflare/ Domain/ Backfill.test.ts

import * as Cloudflare from "@alchemy.run/cloudflare";
import { CloudflareEnvironment } from "@alchemy.run/cloudflare/CloudflareEnvironment";
import { progressOf } from "@alchemy.run/cloudflare/Domain/Backfill";
import { describe, expect } from "@alchemy.run/test";
import * as Test from "@alchemy.run/test/Alchemy";
import * as workflows from "@distilled.cloud/cloudflare/workflows";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as HttpClientRequest from "effect/http/HttpClientRequest";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { requestWorker } from "../Utils/WorkerRequest.ts";
import BenchV1 from "./fixtures/bench/v1.ts";
import BenchV2Short from "./fixtures/bench/v2-short.ts";
import BenchV2 from "./fixtures/bench/v2.ts";
import BenchV3 from "./fixtures/bench/v3.ts";
import LedgerV1 from "./fixtures/catch-up/v1.ts";
import LedgerV2WideBumped from "./fixtures/catch-up/v2-wide-bumped.ts";
import LedgerV2Wide from "./fixtures/catch-up/v2-wide.ts";
import LedgerV2 from "./fixtures/catch-up/v2.ts";

/**
 * How a deploy and a backfill behave together on Cloudflare: a deploy waits
 * for the backfill and reports its progress; it is safe to repeat, to time
 * out and to interrupt (the backfill runs on, and the next deploy waits on it);
 * a version bump rebuilds a view while its backfill runs; and removing a view
 * stops its backfill. Each case seeds accounts, then deploys views over them.
 */

const ACCOUNTS = 1000;
const EVENTS = 5;
// Ten accounts per request, many requests at once.
const SEED_BATCH = 10;

class NotYet extends Data.TaggedError("NotYet")<{
  readonly status: number;
  readonly body: string;
}> {}

const call = (url: string, method: "GET" | "POST", path: string) =>
  requestWorker(
    (method === "GET" ? HttpClientRequest.get : HttpClientRequest.post)(`${url}${path}`).pipe(
      HttpClientRequest.setHeaders({ connection: "close", "cache-control": "no-cache" }),
    ),
  ).pipe(
    Effect.flatMap((res) =>
      res.text.pipe(
        Effect.flatMap((body) =>
          res.status === 200
            ? Effect.succeed(body)
            : Effect.fail(new NotYet({ status: res.status, body })),
        ),
      ),
    ),
    Effect.timeout("2 minutes"),
    Effect.retry({ schedule: Schedule.spaced("2 seconds"), times: 10 }),
  );

const serving = (url: string, version: string) =>
  call(url, "GET", "/version").pipe(
    Effect.filterOrFail(
      (body) => body === version,
      (body) => new NotYet({ status: 200, body }),
    ),
    Effect.retry({ schedule: Schedule.spaced("2 seconds"), times: 90 }),
  );

/** Deploy accounts only, and give each of them its deposits. */
const seeded = (stack: Test.ScratchStack) =>
  Effect.gen(function* () {
    yield* stack.destroy();
    const { worker } = yield* stack.deploy(
      Effect.gen(function* () {
        return { worker: yield* BenchV1 };
      }),
    );
    const url = worker.url!.replace(/\/+$/, "");
    yield* serving(url, "v1");
    yield* Effect.forEach(
      Array.from({ length: ACCOUNTS / SEED_BATCH }, (_, i) => i * SEED_BATCH),
      (from) => call(url, "POST", `/seed/${from}/${SEED_BATCH}/${EVENTS}`),
      { concurrency: 32, discard: true },
    );
    return { url, workerName: worker.workerName };
  });

const deployOf = (stack: Test.ScratchStack, worker: typeof BenchV1) =>
  stack.deploy(
    Effect.gen(function* () {
      return { worker: yield* worker };
    }),
  );

/** The view's state for a few accounts: each has every deposit, once. */
const checkViews = (url: string) =>
  Effect.gen(function* () {
    const expected = Array.from({ length: EVENTS }, (_, n) => n + 1);
    for (const i of [0, ACCOUNTS / 2, ACCOUNTS - 1]) {
      const view = JSON.parse(yield* call(url, "GET", `/view/b${i}`)) as {
        amounts: Array<number>;
      } | null;
      expect(view?.amounts).toEqual(expected);
    }
    const history = JSON.parse(yield* call(url, "GET", "/history")) as { complete: boolean };
    expect(history.complete).toBe(true);
  });

/** Whether a deploy failed because the backfill outlasted its wait. */
const stillRunning = (deploy: Effect.Effect<unknown, unknown, never>) =>
  deploy.pipe(
    Effect.as(""),
    Effect.catchCause((cause) => Effect.succeed(String(cause))),
    Effect.map((failure) => /BackfillStillRunning|still running/.test(failure)),
  );

/** The latest completed instance of the Worker's backfill Workflow. */
const lastWait = (workerName: string) =>
  Effect.gen(function* () {
    const { accountId } = yield* yield* CloudflareEnvironment;
    const all = yield* workflows.listWorkflows.items({ accountId }).pipe(Stream.runCollect);
    const workflow = [...all].find((w) => w.scriptName === workerName);
    expect(workflow).toBeDefined();
    const instances = yield* workflows.listInstances
      .items({ accountId, workflowName: workflow!.name, status: "complete" })
      .pipe(Stream.runCollect);
    const latest = [...instances].sort((a, b) => b.createdOn.localeCompare(a.createdOn))[0];
    expect(latest).toBeDefined();
    return yield* workflows.getInstance({
      accountId,
      workflowName: workflow!.name,
      instanceId: latest!.id,
    });
  });

const { test } = Test.make({ providers: Cloudflare.providers() });

describe.sequential("a deploy and its backfill", () => {
  test.provider(
    "a deploy waits for the backfill and reports its progress, and the same deploy again changes nothing",
    (stack) =>
      Effect.gen(function* () {
        const { url, workerName } = yield* seeded(stack);
        yield* deployOf(stack, BenchV2);
        yield* serving(url, "v2");
        yield* checkViews(url);

        // What the deploy reported as it waited: the run, done, over every account.
        const wait = yield* lastWait(workerName);
        const [run] = progressOf(wait.steps ?? []) ?? [];
        expect(run?.target).toBe("view:Deposits");
        expect(run?.phase).toBe("complete");
        expect(run?.done).toBe(run?.subtrees);
        expect(run?.instances).toBeGreaterThanOrEqual(ACCOUNTS);

        // The same model again: nothing to backfill, nothing to wait for.
        const plan = yield* stack.plan(
          Effect.gen(function* () {
            return { worker: yield* BenchV2 };
          }),
        );
        expect(plan.resources["Ledger.DomainBackfillRun"]?.action).toBe("noop");

        yield* stack.destroy();
      }),
    { timeout: 900_000 },
  );

  test.provider(
    "a deploy that times out leaves the backfill running, and the next deploy waits for it",
    (stack) =>
      Effect.gen(function* () {
        const { url } = yield* seeded(stack);
        expect(yield* stillRunning(deployOf(stack, BenchV2Short))).toBe(true);
        yield* deployOf(stack, BenchV2);
        yield* serving(url, "v2");
        yield* checkViews(url);
        yield* stack.destroy();
      }),
    { timeout: 900_000 },
  );

  test.provider(
    "an interrupted deploy leaves the backfill running, and the next deploy waits for it",
    (stack) =>
      Effect.gen(function* () {
        const { url } = yield* seeded(stack);
        // Interrupt the deploy once the view's code is up and its backfill has begun.
        const deploy = yield* Effect.forkChild(deployOf(stack, BenchV2));
        yield* serving(url, "v2");
        yield* Fiber.interrupt(deploy);
        yield* deployOf(stack, BenchV2);
        yield* checkViews(url);
        yield* stack.destroy();
      }),
    { timeout: 900_000 },
  );

  test.provider(
    "a version bump while a backfill runs rebuilds the view from the history",
    (stack) =>
      Effect.gen(function* () {
        const { url } = yield* seeded(stack);
        expect(yield* stillRunning(deployOf(stack, BenchV2Short))).toBe(true);
        yield* deployOf(stack, BenchV3);
        yield* serving(url, "v3");
        yield* checkViews(url);
        yield* stack.destroy();
      }),
    { timeout: 900_000 },
  );

  test.provider(
    "removing a view while it backfills stops it, and adding it back rebuilds it",
    (stack) =>
      Effect.gen(function* () {
        const { url } = yield* seeded(stack);
        expect(yield* stillRunning(deployOf(stack, BenchV2Short))).toBe(true);
        yield* deployOf(stack, BenchV1);
        yield* serving(url, "v1");
        yield* deployOf(stack, BenchV3);
        yield* serving(url, "v3");
        yield* checkViews(url);
        yield* stack.destroy();
      }),
    { timeout: 900_000 },
  );

  test.provider(
    "a view that reads more sources without a version bump fails the deploy, and a bump rebuilds it",
    (stack) =>
      Effect.gen(function* () {
        yield* stack.destroy();
        const ledger = (worker: typeof LedgerV1) =>
          stack.deploy(
            Effect.gen(function* () {
              return { worker: yield* worker };
            }),
          );
        const { worker } = yield* ledger(LedgerV1);
        const url = worker.url!.replace(/\/+$/, "");
        yield* serving(url, "v1");
        yield* call(url, "POST", "/deposit/a-1/10");
        yield* call(url, "POST", "/deposit/a-2/5");
        yield* ledger(LedgerV2);

        // The timeline's backfill read accounts only: the deploy refuses to wait on it.
        const refused = yield* ledger(LedgerV2Wide).pipe(
          Effect.as(""),
          Effect.catchCause((cause) => Effect.succeed(String(cause))),
        );
        expect(refused).toContain("Bump its version");

        yield* ledger(LedgerV2WideBumped);
        yield* serving(url, "v2-wide-bumped");
        const timeline = JSON.parse(yield* call(url, "GET", "/timeline")) as {
          entries: Array<string>;
        } | null;
        expect([...(timeline?.entries ?? [])].sort()).toEqual(["a-1:10", "a-2:5"]);
        yield* stack.destroy();
      }),
    { timeout: 900_000 },
  );
});