sailfork / t3code-github / .repos/ alchemy-effect/ packages/ integration-tests/ test/ Cloudflare/ Domain/ CatchUp.bench.test.ts

import * as Cloudflare from "@alchemy.run/cloudflare";
import { CloudflareEnvironment } from "@alchemy.run/cloudflare/CloudflareEnvironment";
import { describe, expect } from "@alchemy.run/test";
import * as Test from "@alchemy.run/test/Alchemy";
import { Credentials, formatHeaders } from "@distilled.cloud/cloudflare/Credentials";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as HttpClient from "effect/http/HttpClient";
import * as HttpClientRequest from "effect/http/HttpClientRequest";
import * as Schedule from "effect/Schedule";
import { requestWorker } from "../Utils/WorkerRequest.ts";
import BenchV1 from "./fixtures/bench/v1.ts";
import BenchV2Summary from "./fixtures/bench/v2-summary.ts";
import BenchV2Total from "./fixtures/bench/v2-total.ts";
import BenchV2 from "./fixtures/bench/v2.ts";

/**
 * What a backfill costs and how long a deploy waits for it on Cloudflare. Each
 * run deploys accounts, seeds history, then deploys a view and reads the
 * Durable Object requests and SQLite rows of its backfill from Cloudflare's
 * analytics, printing a `BENCH` line of JSON. Three shapes:
 *
 * - width (`DOMAIN_BENCH_INSTANCES`, default 1000): N accounts with 10 deposits
 *   each, then a view keyed by account;
 * - depth (`DOMAIN_BENCH_DEPTH`, default 1000): one account with N deposits,
 *   then a summary per account;
 * - bank-wide (`DOMAIN_BENCH_TOTAL`, `<accounts>x<events>`, default 100x100):
 *   accounts with deposits, then one total that every account feeds.
 *
 * Sizes are comma-separated; an empty variable skips that shape.
 */

const EVENTS = 10;
// A Worker request runs at most 6 outgoing calls at once: seed in small batches, many at a time.
const SEED_BATCH = 10;
// Deposits to one account per request: one object takes them in order.
const DEPOSIT_BATCH = 1000;
const list = (name: string, fallback: string) =>
  (process.env[name] ?? fallback).split(",").filter((size) => size !== "");
const widths = list("DOMAIN_BENCH_INSTANCES", "1000").map(Number);
const depths = list("DOMAIN_BENCH_DEPTH", "1000").map(Number);
const totals = list("DOMAIN_BENCH_TOTAL", "100x100").map(
  (size) => size.split("x").map(Number) as [number, number],
);

class NotYet extends Data.TaggedError("NotYet")<{
  readonly status: number;
  readonly body: string;
}> {}

const call = (url: string, method: "GET" | "POST", path: string) =>
  requestWorker(
    (method === "GET" ? HttpClientRequest.get : HttpClientRequest.post)(`${url}${path}`).pipe(
      HttpClientRequest.setHeaders({ connection: "close", "cache-control": "no-cache" }),
    ),
  ).pipe(
    Effect.retry({
      while: (error) => error._tag === "HttpClientError",
      schedule: Schedule.spaced("1 second"),
      times: 5,
    }),
    Effect.flatMap((res) =>
      res.text.pipe(
        Effect.flatMap((body) =>
          res.status === 200
            ? Effect.succeed(body)
            : Effect.fail(new NotYet({ status: res.status, body })),
        ),
      ),
    ),
  );

const serving = (url: string, version: string) =>
  call(url, "GET", "/version").pipe(
    Effect.filterOrFail(
      (body) => body === version,
      (body) => new NotYet({ status: 200, body }),
    ),
    Effect.retry({ schedule: Schedule.spaced("2 seconds"), times: 90 }),
  );

interface Usage {
  readonly requests: number;
  readonly byType: Record<string, number>;
  readonly rowsRead: number;
  readonly rowsWritten: number;
  /** Billed duration, in GB-seconds. */
  readonly duration: number;
}

/** The Worker's Durable Object usage between two instants, from Cloudflare's GraphQL analytics. */
const usage = (scriptName: string, from: Date, to: Date) =>
  Effect.gen(function* () {
    const { accountId } = yield* yield* CloudflareEnvironment;
    const headers = formatHeaders(yield* yield* Credentials);
    const client = yield* HttpClient.HttpClient;
    const query = <T>(query: string) =>
      client
        .execute(
          HttpClientRequest.post("https://api.cloudflare.com/client/v4/graphql").pipe(
            HttpClientRequest.setHeaders(headers),
            HttpClientRequest.bodyJsonUnsafe({ query }),
          ),
        )
        .pipe(
          Effect.flatMap((res) => res.json),
          Effect.map(
            (body) => (body as { data: { viewer: { accounts: [T] } } }).data.viewer.accounts[0],
          ),
        );
    const window = `datetime_geq: "${from.toISOString()}", datetime_leq: "${to.toISOString()}"`;
    const { invocations } = yield* query<{
      invocations: ReadonlyArray<{
        sum: { requests: number };
        dimensions: { namespaceId: string; type: string };
      }>;
    }>(`{ viewer { accounts(filter: { accountTag: "${accountId}" }) {
      invocations: durableObjectsInvocationsAdaptiveGroups(limit: 1000, filter: { scriptName: "${scriptName}", ${window} }) {
        sum { requests } dimensions { namespaceId type }
      } } } }`);
    const namespaces = [...new Set(invocations.map((row) => row.dimensions.namespaceId))];
    const { periodic } =
      namespaces.length === 0
        ? { periodic: [] }
        : yield* query<{
            periodic: ReadonlyArray<{
              sum: { rowsRead: number; rowsWritten: number; duration: number };
            }>;
          }>(`{ viewer { accounts(filter: { accountTag: "${accountId}" }) {
            periodic: durableObjectsPeriodicGroups(limit: 1000, filter: { namespaceId_in: ${JSON.stringify(namespaces)}, ${window} }) {
              sum { rowsRead rowsWritten duration } dimensions { namespaceId }
            } } } }`);
    const byType: Record<string, number> = {};
    for (const row of invocations) {
      byType[row.dimensions.type] = (byType[row.dimensions.type] ?? 0) + row.sum.requests;
    }
    return {
      requests: invocations.reduce((total, row) => total + row.sum.requests, 0),
      byType,
      rowsRead: periodic.reduce((total, row) => total + row.sum.rowsRead, 0),
      rowsWritten: periodic.reduce((total, row) => total + row.sum.rowsWritten, 0),
      duration: periodic.reduce((total, row) => total + row.sum.duration, 0),
    } satisfies Usage;
  });

/**
 * Analytics arrive minutes late, and later the more there are: wait five minutes,
 * then read until three reads a minute apart agree. Read before the stack is
 * destroyed: a deleted Worker's rows lose its script name.
 */
const settledUsage = (scriptName: string, from: Date, to: Date) =>
  Effect.gen(function* () {
    yield* Effect.sleep("5 minutes");
    let last = yield* usage(scriptName, from, to);
    let agreed = 0;
    for (let attempt = 0; attempt < 15 && agreed < 2; attempt++) {
      yield* Effect.sleep("1 minute");
      const next = yield* usage(scriptName, from, to);
      agreed =
        next.requests === last.requests && next.rowsWritten === last.rowsWritten ? agreed + 1 : 0;
      last = next;
    }
    return last;
  });

const now = Effect.sync(() => new Date());

const { test } = Test.make({ providers: Cloudflare.providers() });

/** Retry a seed request, logging why: a slow seed is otherwise silent. */
const seedRetry = <A, E extends { readonly _tag: string }, R>(
  path: string,
  effect: Effect.Effect<A, E, R>,
) =>
  effect.pipe(
    // A seed request can hang for many minutes: fail it and send it again (seeds are idempotent).
    Effect.timeout("2 minutes"),
    Effect.tapError((error) =>
      Effect.sync(() =>
        console.log(
          `SEED-RETRY ${path} ${error._tag}${"status" in error ? ` ${String(error.status)}` : ""}${"body" in error ? ` ${String(error.body).slice(0, 120)}` : ""}`,
        ),
      ),
    ),
    Effect.retry({ schedule: Schedule.spaced("2 seconds"), times: 10 }),
  );

const seedAccounts = (url: string, accounts: number, events: number) =>
  Effect.forEach(
    Array.from({ length: Math.ceil(accounts / SEED_BATCH) }, (_, i) => i * SEED_BATCH),
    (from) => {
      const path = `/seed/${from}/${Math.min(SEED_BATCH, accounts - from)}/${events}`;
      return seedRetry(path, call(url, "POST", path));
    },
    { concurrency: 64, discard: true },
  );

const seedOne = (url: string, account: string, events: number) =>
  Effect.forEach(
    Array.from({ length: Math.ceil(events / DEPOSIT_BATCH) }, (_, i) => i * DEPOSIT_BATCH + 1),
    (from) => {
      const path = `/deposit/${account}/${from}/${Math.min(DEPOSIT_BATCH, events - from + 1)}`;
      return seedRetry(path, call(url, "POST", path));
    },
    { concurrency: 4, discard: true },
  );

type Next = typeof BenchV2 | typeof BenchV2Summary | typeof BenchV2Total;

/**
 * Deploy accounts, seed them, deploy `next` and wait for its backfill; check
 * the view, then read the backfill's usage from analytics.
 */
const run = (
  stack: Test.ScratchStack,
  shape: Record<string, unknown> & { readonly events: number; readonly instances: number },
  seed: (url: string) => Effect.Effect<void, unknown, HttpClient.HttpClient>,
  next: Next,
  check: (url: string) => Effect.Effect<void, unknown, HttpClient.HttpClient>,
) =>
  Effect.gen(function* () {
    yield* stack.destroy();
    const { worker } = yield* stack.deploy(
      Effect.gen(function* () {
        return { worker: yield* BenchV1 };
      }),
    );
    const url = worker.url!.replace(/\/+$/, "");
    const scriptName = worker.workerName;
    yield* serving(url, "v1");

    // History before the view exists.
    const seedStart = yield* now;
    yield* seed(url);
    const seedEnd = yield* now;
    // Keep the seed out of the backfill's analytics window.
    yield* Effect.sleep("2 minutes");

    const deployStart = yield* now;
    yield* stack.deploy(
      Effect.gen(function* () {
        return { worker: yield* next };
      }),
    );
    const deployEnd = yield* now;
    yield* serving(url, "v2");

    // The deploy returned once the backfill was done.
    const history = JSON.parse(yield* call(url, "GET", "/history")) as { complete: boolean };
    expect(history.complete).toBe(true);
    yield* check(url);

    const timing = {
      ...shape,
      deploySeconds: +((deployEnd.getTime() - deployStart.getTime()) / 1000).toFixed(1),
      seedSeconds: +((seedEnd.getTime() - seedStart.getTime()) / 1000).toFixed(1),
    };
    // DOMAIN_BENCH_TIME_ONLY: report the deploy's time without waiting minutes for analytics.
    if (process.env.DOMAIN_BENCH_TIME_ONLY) {
      yield* Effect.sync(() => console.log(`BENCH ${JSON.stringify(timing)}`));
      return yield* stack.destroy();
    }
    const settleEnd = new Date(deployEnd.getTime() + 2 * 60_000);
    const seedUsage = yield* settledUsage(scriptName, seedStart, seedEnd);
    const backfill = yield* settledUsage(scriptName, deployStart, settleEnd);
    const result = {
      ...timing,
      backfill,
      seed: seedUsage,
      requestsPerInstance: +(backfill.requests / shape.instances).toFixed(2),
      rowsWrittenPerEvent: +(backfill.rowsWritten / shape.events).toFixed(2),
      rowsReadPerEvent: +(backfill.rowsRead / shape.events).toFixed(2),
    };
    yield* Effect.sync(() => console.log(`BENCH ${JSON.stringify(result)}`));
    yield* stack.destroy();
  });

const viewOf = (url: string, key: string) =>
  call(url, "GET", `/view/${key}`).pipe(Effect.map((body) => JSON.parse(body)));

// Each run has its own Worker and stack: run them at once, or a full sweep takes hours.
describe.concurrent("live backfill benchmarks", () => {
  for (const instances of widths) {
    test.provider.skipIf(!!process.env.FAST)(
      `live: backfill of ${instances} instances with ${EVENTS} events each`,
      (stack) =>
        run(
          stack,
          { shape: "width", instances, events: instances * EVENTS },
          (url) => seedAccounts(url, instances, EVENTS),
          BenchV2,
          (url) =>
            Effect.gen(function* () {
              const expected = Array.from({ length: EVENTS }, (_, n) => n + 1);
              for (const i of [0, Math.floor(instances / 2), instances - 1]) {
                const account = JSON.parse(yield* call(url, "GET", `/account/b${i}`)) as {
                  balance: number;
                };
                expect(account.balance).toBe((EVENTS * (EVENTS + 1)) / 2);
                expect(
                  ((yield* viewOf(url, `b${i}`)) as { amounts: Array<number> }).amounts,
                ).toEqual(expected);
              }
            }),
        ),
      { timeout: 7_200_000 },
    );
  }

  for (const events of depths) {
    test.provider.skipIf(!!process.env.FAST)(
      `live: backfill of one account with ${events} events`,
      (stack) =>
        run(
          stack,
          { shape: "depth", instances: 1, events },
          (url) => seedOne(url, "deep", events),
          BenchV2Summary,
          (url) =>
            Effect.gen(function* () {
              expect(yield* viewOf(url, "deep")).toEqual({
                count: events,
                total: (events * (events + 1)) / 2,
              });
            }),
        ),
      { timeout: 7_200_000 },
    );
  }

  for (const [accounts, events] of totals) {
    test.provider.skipIf(!!process.env.FAST)(
      `live: backfill of a bank-wide total over ${accounts} accounts with ${events} events each`,
      (stack) =>
        run(
          stack,
          { shape: "total", instances: accounts, events: accounts * events },
          (url) => seedAccounts(url, accounts, events),
          BenchV2Total,
          (url) =>
            Effect.gen(function* () {
              expect(yield* viewOf(url, "bank")).toEqual({
                count: accounts * events,
                total: (accounts * events * (events + 1)) / 2,
              });
            }),
        ),
      { timeout: 7_200_000 },
    );
  }
});