import * as Cloudflare from "@alchemy.run/cloudflare";
import { describe, expect } from "@alchemy.run/test";
import * as Test from "@alchemy.run/test/Alchemy";
import * as Alchemy from "alchemy";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as HttpClientRequest from "effect/http/HttpClientRequest";
import * as Path from "effect/Path";
import * as Ref from "effect/Ref";
import * as Schedule from "effect/Schedule";
import { requestWorker } from "../Utils/WorkerRequest.ts";
import BenchV1 from "./fixtures/bench/v1.ts";
/**
* How long a deploy waits for a backfill as the keyspace grows, on one stack
* that stays deployed, so each step doesn't seed again:
*
* - `DOMAIN_SCALE=seed`: deploy accounts and seed `DOMAIN_SCALE_ACCOUNTS` of
* them (default 100,000), 10 deposits each;
* - `DOMAIN_SCALE=rebuild`: deploy a view keyed by account at version
* `DOMAIN_SCALE_VERSION` (each run a new one, so it is rebuilt from the
* history), print a `SCALE` line of JSON, and check a few keys;
* - `DOMAIN_SCALE=destroy`: remove the stack.
*/
const step = process.env.DOMAIN_SCALE;
const accounts = Number(process.env.DOMAIN_SCALE_ACCOUNTS ?? "100000");
const version = Number(process.env.DOMAIN_SCALE_VERSION ?? "1");
const EVENTS = 10;
// Ten accounts per request, many requests at once.
const SEED_BATCH = 10;
const state = Cloudflare.state();
const { test, deploy, destroy } = Test.make({ providers: Cloudflare.providers(), state });
class NotYet extends Data.TaggedError("NotYet")<{
readonly status: number;
readonly body: string;
}> {}
const call = (url: string, method: "GET" | "POST", path: string) =>
requestWorker(
(method === "GET" ? HttpClientRequest.get : HttpClientRequest.post)(`${url}${path}`).pipe(
HttpClientRequest.setHeaders({ connection: "close", "cache-control": "no-cache" }),
),
).pipe(
Effect.flatMap((res) =>
res.text.pipe(
Effect.flatMap((body) =>
res.status === 200
? Effect.succeed(body)
: Effect.fail(new NotYet({ status: res.status, body })),
),
),
),
// A request can hang for minutes: send it again (every route here is idempotent).
Effect.timeout("2 minutes"),
Effect.retry({ schedule: Schedule.spaced("2 seconds"), times: 10 }),
);
const serving = (url: string, expected: string) =>
call(url, "GET", "/version").pipe(
Effect.filterOrFail(
(body) => body === expected,
(body) => new NotYet({ status: 200, body }),
),
Effect.retry({ schedule: Schedule.spaced("2 seconds"), times: 90 }),
);
const now = Effect.sync(() => Date.now());
const stackOf = <W extends typeof BenchV1>(worker: W) =>
Alchemy.Stack(
`DomainScale-${accounts}`,
{ providers: Cloudflare.providers(), state },
Effect.gen(function* () {
const deployed = yield* worker;
return { url: deployed.url.as<string>(), workerName: deployed.workerName };
}),
);
/** The view at `at`, from a fixture generated for it: the bundle carries the version. */
const versioned = (at: number) =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const dir = path.join(import.meta.dirname, "fixtures", "bench", "generated");
yield* fs.makeDirectory(dir, { recursive: true });
const file = path.join(dir, `v2-${at}.ts`);
yield* fs.writeFileString(
file,
`import * as Cloudflare from "@alchemy.run/cloudflare";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Domain from "@alchemy.run/domain";
import { Account, Deposits } from "../../catch-up/ledger.ts";
import { benchWorker, hosted } from "../worker.ts";
/** The view at version ${at}. */
class DepositsAt extends Domain.View("Deposits", { ...Deposits.definition, version: ${at} }) {}
class AfterAt extends Domain.Model("Ledger", { aggregates: [Account], views: [DepositsAt] }) {}
/** Generated by CatchUp.scale.test.ts: the view at version ${at}. */
export default class DomainBenchWorker extends Cloudflare.Worker<DomainBenchWorker>()(
"DomainBenchWorker",
{ main: import.meta.url },
benchWorker("v2", { kind: "deposits", view: DepositsAt as typeof Deposits }).pipe(
Effect.provide(hosted(AfterAt.layer(Layer.empty))),
),
) {}
`,
);
const module = yield* Effect.promise(() => import(file));
return module.default as typeof BenchV1;
});
describe("domain backfill at scale", () => {
test.skipIf(step !== "seed")(
`seed ${accounts} accounts`,
Effect.gen(function* () {
const { url } = yield* deploy(stackOf(BenchV1));
const base = url.replace(/\/+$/, "");
yield* serving(base, "v1");
const batches = Math.ceil(accounts / SEED_BATCH);
const done = yield* Ref.make(0);
const started = yield* now;
yield* Effect.forEach(
Array.from({ length: batches }, (_, i) => i * SEED_BATCH),
(from) =>
call(
base,
"POST",
`/seed/${from}/${Math.min(SEED_BATCH, accounts - from)}/${EVENTS}`,
).pipe(
Effect.andThen(Ref.updateAndGet(done, (n) => n + 1)),
Effect.tap((n) =>
n % 500 === 0
? Effect.flatMap(now, (at) =>
Effect.sync(() =>
console.log(
`SEED ${n * SEED_BATCH}/${accounts} ${((at - started) / 1000).toFixed(0)}s`,
),
),
)
: Effect.void,
),
),
{ concurrency: 64, discard: true },
);
const ended = yield* now;
yield* Effect.sync(() =>
console.log(`SEEDED ${JSON.stringify({ accounts, seconds: (ended - started) / 1000 })}`),
);
}),
{ timeout: 7_200_000 },
);
test.skipIf(step !== "rebuild")(
`rebuild a view over ${accounts} accounts at version ${version}`,
Effect.gen(function* () {
const worker = yield* versioned(version);
const started = yield* now;
const { url } = yield* deploy(stackOf(worker));
const ended = yield* now;
const base = url.replace(/\/+$/, "");
yield* serving(base, "v2");
const history = JSON.parse(yield* call(base, "GET", "/history")) as { complete: boolean };
const expected = Array.from({ length: EVENTS }, (_, n) => n + 1);
for (const i of [0, Math.floor(accounts / 2), accounts - 1]) {
const view = JSON.parse(yield* call(base, "GET", `/view/b${i}`)) as {
amounts: Array<number>;
} | null;
expect(view?.amounts).toEqual(expected);
}
expect(history.complete).toBe(true);
yield* Effect.sync(() =>
console.log(
`SCALE ${JSON.stringify({ accounts, version, deploySeconds: (ended - started) / 1000 })}`,
),
);
}),
{ timeout: 7_200_000 },
);
test.skipIf(step !== "destroy")(
"destroy",
Effect.gen(function* () {
yield* destroy(stackOf(BenchV1));
}),
{ timeout: 1_800_000 },
);
});