sailfork / t3code-github / .repos/ alchemy-effect/ packages/ integration-tests/ test/ Cloudflare/ Workers/ AlarmCallbackScheduler.test.ts

import { describe, expect, it } from "@alchemy.run/test";
import { CallbackError, type Callback, type CallbackOptions } from "alchemy/Callback";
import { RuntimeContext } from "alchemy/RuntimeContext";
import * as Cause from "effect/Cause";
import * as Clock from "effect/Clock";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as Fiber from "effect/Fiber";
import * as Logger from "effect/Logger";
import * as TestClock from "effect/testing/TestClock";
import { makeAlarmScheduler } from "./fixtures/alarm-scheduler.ts";

describe(
  "durable callback scheduling policy",
  {
    tags: ["unit", "local", "provider:cloudflare", "provider:cloudflare:worker"],
  },
  () => {
    it.effect("acknowledges a successful job after persisting its recovery wake", () =>
      Effect.gen(function* () {
        const fixture = yield* makeAlarmScheduler;
        const delivered: string[] = [];
        const callback = yield* fixture.register("archive", (payload: string) =>
          Effect.gen(function* () {
            expect(fixture.alarm!).toBeGreaterThan(yield* Clock.currentTimeMillis);
            delivered.push(payload);
          }),
        );
        fixture.seal();
        yield* callback.schedule("a", { after: 0, payload: "saved" });
        yield* fixture.fire;
        expect(delivered).toEqual(["saved"]);
        expect(fixture.alarm).toBeNull();
      }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    it.effect("backs off failures durably across instance reconstruction", () =>
      Effect.gen(function* () {
        const fixture = yield* makeAlarmScheduler;
        const register = () =>
          fixture.register("broken", () => Effect.fail("unavailable"), {
            retry: { delay: "1 second" },
          });
        const callback = yield* register();
        fixture.seal();
        yield* callback.schedule("a", { after: 0, payload: null });
        yield* fixture.fire;
        expect(fixture.alarm! - (yield* Clock.currentTimeMillis)).toBe(1_000);
        fixture.restart();
        yield* register();
        fixture.seal();
        yield* fixture.fire;
        expect(fixture.alarm! - (yield* Clock.currentTimeMillis)).toBe(2_000);
      }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    for (const changeId of [false, true]) {
      it.effect(
        `bounds successful self-rearms${changeId ? " across cancellation and changed IDs" : ""}`,
        () =>
          Effect.gen(function* () {
            const fixture = yield* makeAlarmScheduler;
            let deliveries = 0;
            const callback: Callback<null> = yield* fixture.register(
              "loop",
              () =>
                Effect.gen(function* () {
                  yield* callback.cancel(String(changeId ? deliveries : 0));
                  deliveries++;
                  yield* callback.schedule(String(changeId ? deliveries : 0), {
                    after: 0,
                    payload: null,
                  });
                }),
              { retry: { delay: "1 second" } },
            );
            fixture.seal();
            yield* callback.schedule("0", { after: 0, payload: null });
            for (const delay of [
              1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 64_000, 3_600_000, 3_600_000,
            ]) {
              yield* fixture.fire;
              expect(fixture.alarm! - (yield* Clock.currentTimeMillis)).toBe(delay);
            }
            expect(deliveries).toBe(9);
          }).pipe(Effect.provide(RuntimeContext.phantom)),
      );
    }

    it.effect(
      "spaces retries from failure completion when a handler outlasts its recovery delay",
      () =>
        Effect.gen(function* () {
          const fixture = yield* makeAlarmScheduler;
          const callback = yield* fixture.register(
            "slow",
            () =>
              Effect.gen(function* () {
                yield* TestClock.adjust("10 seconds");
                return yield* Effect.fail("still unavailable");
              }),
            { retry: { delay: "1 second" } },
          );
          fixture.seal();
          yield* callback.schedule("a", { after: 0, payload: null });
          yield* fixture.fire;
          expect(fixture.alarm! - (yield* Clock.currentTimeMillis)).toBe(1_000);
        }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    it.effect("parks repeated failures and preserves parking after reconstruction", () =>
      Effect.gen(function* () {
        const fixture = yield* makeAlarmScheduler;
        const register = () =>
          fixture.register("broken", () => Effect.fail("unavailable"), {
            retry: { delay: "1 second" },
          });
        const callback = yield* register();
        fixture.seal();
        yield* callback.schedule("a", { after: 0, payload: null });
        for (let attempt = 0; attempt < 8; attempt++) yield* fixture.fire;
        expect(fixture.alarm! - (yield* Clock.currentTimeMillis)).toBe(3_600_000);
        fixture.restart();
        yield* register();
        fixture.seal();
        yield* fixture.fire;
        expect(fixture.alarm! - (yield* Clock.currentTimeMillis)).toBe(3_600_000);
      }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    it.effect("enforces a one-second floor for subsecond retry delays", () =>
      Effect.gen(function* () {
        const fixture = yield* makeAlarmScheduler;
        const callback = yield* fixture.register("broken", () => Effect.fail("unavailable"), {
          retry: { delay: "1 millis" },
        });
        fixture.seal();
        yield* callback.schedule("a", { after: 0, payload: null });
        yield* fixture.fire;
        expect(fixture.alarm! - (yield* Clock.currentTimeMillis)).toBe(1_000);
      }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    it.effect("rejects a detached self-rearm after the callback invocation ends", () =>
      Effect.gen(function* () {
        const fixture = yield* makeAlarmScheduler;
        const release = yield* Deferred.make<void>();
        let detached: Fiber.Fiber<void, unknown> | undefined;
        const callback: Callback<null> = yield* fixture.register("detached", () =>
          Effect.gen(function* () {
            detached = yield* Deferred.await(release).pipe(
              Effect.andThen(() => callback.schedule("a", { after: 0, payload: null })),
              Effect.forkDetach,
            );
          }),
        );
        fixture.seal();
        yield* callback.schedule("a", { after: 0, payload: null });
        yield* fixture.fire;
        yield* Deferred.succeed(release, undefined);
        expect(Exit.isFailure(yield* Fiber.await(detached!))).toBe(true);
        expect(fixture.alarm).toBeNull();
      }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    it.effect("continues unrelated due work when a callback fails", () =>
      Effect.gen(function* () {
        const fixture = yield* makeAlarmScheduler;
        let delivered = false;
        const broken = yield* fixture.register("a", () => Effect.fail("unavailable"));
        const healthy = yield* fixture.register("b", () =>
          Effect.sync(() => {
            delivered = true;
          }),
        );
        fixture.seal();
        yield* broken.schedule("a", { after: 0, payload: null });
        yield* healthy.schedule("b", { after: 0, payload: null });
        yield* fixture.fire;
        expect(delivered).toBe(true);
      }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    it.effect(
      "preserves the requested deadline and inspects status without decoding payloads",
      () =>
        Effect.gen(function* () {
          const fixture = yield* makeAlarmScheduler;
          const callback = yield* fixture.register("broken", () => Effect.fail("unavailable"), {
            retry: { delay: "1 second" },
          });
          fixture.seal();
          yield* callback.schedule("a", {
            at: 15_000,
            payload: null,
            progress: 0,
          });
          expect(yield* callback.getStatus("a")).toEqual({
            scheduledAt: 15_000,
            retryAt: undefined,
            attempts: 0,
            parked: false,
            progress: 0,
          });
          yield* fixture.fire;
          const status = yield* callback.getStatus("a");
          expect(status).toEqual({
            scheduledAt: 15_000,
            retryAt: 16_000,
            attempts: 1,
            parked: false,
            progress: 0,
          });
          fixture.db.run("UPDATE alchemy_alarm_callbacks SET payload = 'invalid json'");
          expect(yield* callback.getStatus("a")).toEqual(status);
          yield* callback.cancel("a");
          expect(yield* callback.getStatus("a")).toBeUndefined();
          expect(fixture.alarm).toBeNull();
        }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    it.effect(
      "keeps parked jobs parked when the budget grows and resumes only on new progress or external enrollment",
      () =>
        Effect.gen(function* () {
          const fixture = yield* makeAlarmScheduler;
          const register = (maxAttempts: number) =>
            fixture.register("broken", () => Effect.fail("unavailable"), {
              retry: {
                delay: "2 seconds",
                maxAttempts,
                parkedDelay: "2 hours",
              },
            });
          let callback = yield* register(2);
          fixture.seal();
          yield* callback.schedule("a", {
            after: 0,
            payload: null,
            progress: 1,
          });
          yield* fixture.fire;
          yield* fixture.fire;
          const parked = yield* callback.getStatus("a");
          expect(parked?.parked).toBe(true);
          expect(parked?.attempts).toBe(2);
          expect(fixture.alarm! - (yield* Clock.currentTimeMillis)).toBe(7_200_000);
          fixture.restart();
          callback = yield* register(20);
          fixture.seal();
          yield* fixture.fire;
          expect((yield* callback.getStatus("a"))?.attempts).toBe(2);
          expect((yield* callback.getStatus("a"))?.parked).toBe(true);
          const nextWake = fixture.alarm;
          for (const progress of [0, 1]) {
            yield* callback.schedule("a", {
              after: 0,
              payload: "stale",
              progress,
            });
            expect(fixture.alarm).toBe(nextWake);
          }
          yield* callback.schedule("a", {
            after: 0,
            payload: "new",
            progress: 2,
          });
          expect(yield* callback.getStatus("a")).toEqual({
            scheduledAt: yield* Clock.currentTimeMillis,
            retryAt: undefined,
            attempts: 0,
            parked: false,
            progress: 2,
          });
          yield* fixture.fire;
          yield* callback.schedule("a", {
            after: 0,
            payload: "explicit recovery",
          });
          expect((yield* callback.getStatus("a"))?.attempts).toBe(0);
          expect((yield* callback.getStatus("a"))?.progress).toBe(2);
        }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    it.effect("weighs another callback's cursor without letting it hide a schedule", () =>
      Effect.gen(function* () {
        const fixture = yield* makeAlarmScheduler;
        const delivered: string[] = [];
        const target = yield* fixture.register(
          "target",
          (id: string) => Effect.sync(() => delivered.push(id)),
          { retry: { delay: "1 second" } },
        );
        const source = yield* fixture.register(
          "source",
          () =>
            Effect.gen(function* () {
              yield* target.schedule("behind", { after: 0, payload: "behind", progress: 1 });
              yield* target.schedule("past", { after: 0, payload: "past", progress: 101 });
            }),
          { retry: { delay: "1 second" } },
        );
        fixture.seal();
        yield* source.schedule("s", { after: 0, payload: null, progress: 100 });
        yield* fixture.fire;
        // Behind the source's cursor: scheduled under its own cursor, with the
        // source's budget. Past it: progress, so a fresh budget.
        expect(yield* target.getStatus("behind")).toMatchObject({ attempts: 1, progress: 1 });
        expect(yield* target.getStatus("past")).toMatchObject({ attempts: 0, progress: 101 });
        yield* fixture.fire;
        expect(delivered).toEqual(["past"]);
        yield* fixture.fire;
        expect(delivered).toEqual(["past", "behind"]);
      }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    for (const changeId of [false, true]) {
      it.effect(
        `resets the budget for committed forward progress${changeId ? " across IDs" : ""}`,
        () =>
          Effect.gen(function* () {
            const fixture = yield* makeAlarmScheduler;
            fixture.db.exec(
              "CREATE TABLE source (version INTEGER NOT NULL); INSERT INTO source VALUES (0)",
            );
            let version = 0;
            const callback: Callback<null> = yield* fixture.register(
              "progress",
              () =>
                fixture.state.storage.transaction(
                  Effect.gen(function* () {
                    version++;
                    yield* fixture.state.storage.sql.exec("UPDATE source SET version = ?", version);
                    yield* callback.schedule(String(changeId ? version : 0), {
                      after: 0,
                      payload: null,
                      progress: version,
                    });
                  }),
                ),
              { retry: { delay: "1 second", maxAttempts: 2 } },
            );
            fixture.seal();
            yield* callback.schedule("0", {
              after: 0,
              payload: null,
              progress: 0,
            });
            for (let i = 1; i <= 8; i++) {
              yield* fixture.fire;
              const status = yield* callback.getStatus(String(changeId ? i : 0));
              expect(status?.attempts).toBe(0);
              expect(status?.parked).toBe(false);
              expect(status?.progress).toBe(i);
              expect(fixture.alarm! - (yield* Clock.currentTimeMillis)).toBe(1);
            }
            expect(fixture.db.query("SELECT version FROM source").get()).toEqual({ version: 8 });
          }).pipe(Effect.provide(RuntimeContext.phantom)),
      );
    }

    it.effect(
      "rolls back application progress and self-rearms together without undoing the persisted recovery attempt",
      () =>
        Effect.gen(function* () {
          const fixture = yield* makeAlarmScheduler;
          fixture.db.exec(
            "CREATE TABLE source (version INTEGER NOT NULL); INSERT INTO source VALUES (0)",
          );
          const callback: Callback<null> = yield* fixture.register(
            "rollback",
            () =>
              fixture.state.storage.transaction(
                Effect.gen(function* () {
                  yield* fixture.state.storage.sql.exec("UPDATE source SET version = 1");
                  yield* callback.schedule("a", {
                    after: 0,
                    payload: null,
                    progress: 1,
                  });
                  return yield* Effect.fail("rollback progress");
                }),
              ),
            { retry: { delay: "1 second", maxAttempts: 1 } },
          );
          fixture.seal();
          yield* callback.schedule("a", {
            at: 15_000,
            payload: null,
            progress: 0,
          });
          yield* fixture.fire;
          expect(fixture.db.query("SELECT version FROM source").get()).toEqual({
            version: 0,
          });
          expect(yield* callback.getStatus("a")).toEqual({
            scheduledAt: 15_000,
            retryAt: 3_615_000,
            attempts: 1,
            parked: true,
            progress: 0,
          });
          expect(fixture.alarm).toBe(3_615_000);
        }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    it.effect(
      "charges multiple self-rearms followed by failure only once and preserves the last replacement",
      () =>
        Effect.gen(function* () {
          const fixture = yield* makeAlarmScheduler;
          const callback: Callback<string> = yield* fixture.register(
            "replace",
            () =>
              Effect.gen(function* () {
                yield* callback.schedule("a", {
                  after: "1 minute",
                  payload: "first",
                });
                yield* callback.schedule("a", {
                  after: "2 minutes",
                  payload: "last",
                });
                return yield* Effect.fail("old attempt failed");
              }),
            { retry: { delay: "1 second" } },
          );
          fixture.seal();
          yield* callback.schedule("a", { after: 0, payload: "initial" });
          yield* fixture.fire;
          const now = yield* Clock.currentTimeMillis;
          expect(yield* callback.getStatus("a")).toEqual({
            scheduledAt: now + 120_000,
            retryAt: now + 1_000,
            attempts: 1,
            parked: false,
            progress: undefined,
          });
          expect(fixture.alarm).toBe(now + 120_000);
          expect(fixture.db.query("SELECT payload FROM alchemy_alarm_callbacks").get()).toEqual({
            payload: '"last"',
          });
        }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    it.effect("does not overwrite an external replacement when an older delivery fails", () =>
      Effect.gen(function* () {
        const fixture = yield* makeAlarmScheduler;
        const entered = yield* Deferred.make<void>();
        const release = yield* Deferred.make<void>();
        const callback = yield* fixture.register(
          "replace",
          () =>
            Effect.gen(function* () {
              yield* Deferred.succeed(entered, undefined);
              yield* Deferred.await(release);
              return yield* Effect.fail("stale failure");
            }),
          { retry: { delay: "1 second" } },
        );
        fixture.seal();
        yield* callback.schedule("a", {
          after: 0,
          payload: null,
          progress: 1,
        });
        const delivery = yield* fixture.fire.pipe(Effect.forkChild);
        yield* Deferred.await(entered);
        yield* callback.schedule("a", {
          after: "5 minutes",
          payload: "replacement",
          progress: 2,
        });
        const replacement = yield* callback.getStatus("a");
        const wake = fixture.alarm;
        yield* Deferred.succeed(release, undefined);
        yield* Fiber.join(delivery);
        expect(yield* callback.getStatus("a")).toEqual(replacement);
        expect(fixture.alarm).toBe(wake);
      }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    it.effect("parks missing handlers and delivers retained work when the handler returns", () =>
      Effect.gen(function* () {
        const fixture = yield* makeAlarmScheduler;
        const callback = yield* fixture.register("removed", () => Effect.void);
        fixture.seal();
        yield* callback.schedule("a", { after: 0, payload: null });
        fixture.restart();
        fixture.seal();
        for (let i = 0; i < 8; i++) yield* fixture.fire;
        expect((yield* callback.getStatus("a"))?.parked).toBe(true);
        expect(fixture.alarm! - (yield* Clock.currentTimeMillis)).toBe(3_600_000);
        fixture.restart();
        let delivered = false;
        yield* fixture.register("removed", () =>
          Effect.sync(() => {
            delivered = true;
          }),
        );
        fixture.seal();
        yield* fixture.fire;
        expect(delivered).toBe(true);
        expect(yield* callback.getStatus("a")).toBeUndefined();
        expect(fixture.alarm).toBeNull();
      }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    it.effect(
      "migrates version-one callback rows atomically while preserving their native wake",
      () =>
        Effect.gen(function* () {
          const fixture = yield* makeAlarmScheduler;
          fixture.db.exec(`
        CREATE TABLE alchemy_alarm_schema (id INTEGER PRIMARY KEY, version INTEGER NOT NULL);
        INSERT INTO alchemy_alarm_schema VALUES (1, 1);
        CREATE TABLE alchemy_scheduled_events (id TEXT PRIMARY KEY, run_at INTEGER NOT NULL, repeat_ms INTEGER, payload TEXT NOT NULL);
        CREATE TABLE alchemy_alarm_callbacks (
          callback TEXT NOT NULL, id TEXT NOT NULL, version TEXT NOT NULL,
          run_at INTEGER NOT NULL, payload TEXT NOT NULL, PRIMARY KEY (callback, id)
        );
        CREATE INDEX idx_alchemy_alarm_callbacks_run_at ON alchemy_alarm_callbacks (run_at, callback, id);
        INSERT INTO alchemy_alarm_callbacks VALUES ('legacy', 'a', 'existing-version', 15000, 'null');
      `);
          yield* Effect.promise(() => fixture.raw.storage.setAlarm(15_000));
          const before = fixture.db.query("SELECT * FROM sqlite_master ORDER BY name").all();
          const callback = yield* fixture.register("legacy", () => Effect.void);
          fixture.seal();
          const rollback = yield* fixture.state.storage
            .transaction(
              Effect.gen(function* () {
                yield* callback.getStatus("a");
                return yield* Effect.fail("rollback migration");
              }),
            )
            .pipe(Effect.exit);
          expect(Exit.isFailure(rollback)).toBe(true);
          expect(fixture.db.query("SELECT * FROM sqlite_master ORDER BY name").all()).toEqual(
            before,
          );
          expect(fixture.alarm).toBe(15_000);
          expect(yield* callback.getStatus("a")).toEqual({
            scheduledAt: 15_000,
            retryAt: undefined,
            attempts: 0,
            parked: false,
            progress: undefined,
          });
          expect(fixture.db.query("SELECT version FROM alchemy_alarm_schema").get()).toEqual({
            version: 2,
          });
          expect(
            fixture.db.query("SELECT version, payload FROM alchemy_alarm_callbacks").get(),
          ).toEqual({ version: "existing-version", payload: "null" });
          expect(fixture.alarm).toBe(15_000);
          yield* fixture.fire;
          expect(fixture.alarm).toBeNull();
        }).pipe(Effect.provide(RuntimeContext.phantom)),
    );

    for (const retry of [
      { delay: 0 },
      { delay: Infinity },
      { maxAttempts: 0 },
      { maxAttempts: 1.5 },
      { maxAttempts: Infinity },
      { parkedDelay: "59 minutes" },
      { parkedDelay: Infinity },
      { delay: "2 hours", parkedDelay: "1 hour" },
    ] satisfies NonNullable<CallbackOptions["retry"]>[]) {
      it.effect(`rejects invalid retry policy ${JSON.stringify(retry)}`, () =>
        Effect.gen(function* () {
          const fixture = yield* makeAlarmScheduler;
          const exit = yield* fixture
            .register("invalid", () => Effect.void, { retry })
            .pipe(Effect.exit);
          expect(Exit.isFailure(exit)).toBe(true);
          if (Exit.isFailure(exit)) expect(Cause.squash(exit.cause)).toBeInstanceOf(CallbackError);
          expect(fixture.alarm).toBeNull();
        }).pipe(Effect.provide(RuntimeContext.phantom)),
      );
    }

    it.effect("rejects invalid source cursors without replacing pending work", () =>
      Effect.gen(function* () {
        const fixture = yield* makeAlarmScheduler;
        const callback = yield* fixture.register("invalid", () => Effect.void);
        fixture.seal();
        yield* callback.schedule("a", { after: "1 minute", payload: null });
        const before = yield* callback.getStatus("a");
        for (const progress of [-1, 1.5, Infinity, Number.MAX_SAFE_INTEGER + 1]) {
          const exit = yield* callback
            .schedule("a", { after: 0, payload: null, progress })
            .pipe(Effect.exit);
          expect(Exit.isFailure(exit)).toBe(true);
          expect(yield* callback.getStatus("a")).toEqual(before);
        }
      }).pipe(Effect.provide(RuntimeContext.phantom)),
    );
    it.effect("reports committed parking once even when a later rearm rolls back", () => {
      const reports: unknown[][] = [];
      const logger = Logger.make(({ message }) => {
        if (Array.isArray(message) && message[0] === "Durable Object alarm callback parked") {
          reports.push(message);
        }
      });
      return Effect.gen(function* () {
        const fixture = yield* makeAlarmScheduler;
        const callback: Callback<string> = yield* fixture.register(
          "PRIVATE_CALLBACK",
          () =>
            fixture.state.storage.transaction(
              Effect.gen(function* () {
                yield* callback.schedule("PRIVATE_ID", {
                  after: 0,
                  payload: "PRIVATE_PAYLOAD",
                });
                return yield* Effect.fail("rollback rearm");
              }),
            ),
          { retry: { delay: "1 second", maxAttempts: 1 } },
        );
        fixture.seal();
        yield* callback.schedule("PRIVATE_ID", {
          after: 0,
          payload: "PRIVATE_PAYLOAD",
        });
        yield* fixture.fire;
        expect(reports).toHaveLength(1);
        expect(reports[0]).toEqual([
          "Durable Object alarm callback parked",
          { attempts: 1, retryAt: fixture.alarm },
        ]);
        yield* fixture.fire;
        expect(reports).toHaveLength(1);
      }).pipe(
        Effect.provideService(Logger.CurrentLoggers, new Set([logger])),
        Effect.provide(RuntimeContext.phantom),
      );
    });
  },
);