sailfork / t3code-github / apps/ server/ src/ scheduledTasks/ ScheduledTaskService.webhook.test.ts

// @effect-diagnostics-next-line nodeBuiltinImport:off -- Effect's Crypto has no createHmac.
import * as NodeCrypto from "node:crypto";

import * as NodePlatformCrypto from "@effect/platform-node/NodeCrypto";
import { assert, it } from "@effect/vitest";
import {
  type ManagedPolicyDocument,
  ScheduledTaskUpsertInput,
  ScheduledTaskWebhookDeliveryId,
  SecretRequestError,
} from "@t3tools/contracts";
import * as DateTime from "effect/DateTime";
import * as SecureRandom from "effect/crypto/SecureRandom";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as Fiber from "effect/Fiber";
import * as Layer from "effect/Layer";
import * as Queue from "effect/Queue";
import * as EffectScheduler from "effect/Scheduler";
import * as Schema from "effect/Schema";
import * as SqlClient from "effect/sql/SqlClient";
import * as TestClock from "effect/testing/TestClock";
import * as Tracer from "effect/Tracer";

import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
import * as ManagedPolicy from "../policy/ManagedPolicy.ts";
import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
import * as SecretRequests from "../secrets/SecretRequests.ts";
import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts";
import * as SqlitePersistence from "../persistence/Sqlite.ts";
import * as Scheduler from "../scheduling/Scheduler.ts";
import * as ScheduledTaskService from "./ScheduledTaskService.ts";
import { renderWebhookPrompt } from "./webhookTemplate.ts";

const decodeUpsertInput = Schema.decodeUnknownEffect(ScheduledTaskUpsertInput);

it.effect(
  "preserves omitted dedupe settings, releases duplicate claims, and retains coalesced claims",
  () =>
    withService(({ service, sql, launches }) =>
      Effect.gen(function* () {
        const dedupe = {
          replayKey: "{{headers.delivery-id}}",
          coalesceKey: "{{body.group}}",
          coalesceWindowSeconds: 10,
        };
        const { task } = yield* service.upsert(
          yield* webhookTaskInput({ schedule: { type: "webhook", dedupe } }),
        );
        const updated = yield* service.upsert(yield* webhookTaskInput());
        assert.deepEqual(
          updated.task.schedule.type === "webhook" && updated.task.schedule.dedupe,
          dedupe,
        );
        const request = (id: string, bodyText = "{}", relayDeliveryId?: string) =>
          requestFor(task, {
            headers: { "delivery-id": id },
            bodyText,
            body: new TextEncoder().encode(bodyText),
            ...(relayDeliveryId === undefined ? {} : { relayDeliveryId }),
          });
        yield* service.triggerWebhook(request("A"));
        yield* Queue.take(launches);
        yield* service.triggerWebhook(request("B1", '{"group":"B"}'));
        for (let i = 0; i < 5; i++) {
          const duplicate = yield* service.triggerWebhook(request("A", "{}", `duplicate-${i}`));
          assert.equal(duplicate._tag === "accepted" && duplicate.outcome, "duplicate");
          const coalesced = yield* service.triggerWebhook(
            request(`B${i + 2}`, '{"group":"B"}', `coalesced-${i}`),
          );
          assert.equal(coalesced._tag === "accepted" && coalesced.outcome, "coalesced");
        }
        const claims = yield* sql<{
          relay_delivery_id: string;
        }>`SELECT relay_delivery_id FROM scheduled_task_webhook_relay_deliveries ORDER BY relay_delivery_id`;
        assert.deepEqual(
          claims.map((claim) => claim.relay_delivery_id),
          Array.from({ length: 5 }, (_, i) => `coalesced-${i}`),
        );
        assert.isTrue(
          (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.every(
            (delivery) => delivery.outcome !== "duplicate" && delivery.outcome !== "coalesced",
          ),
        );
        const fresh = yield* service.triggerWebhook(request("C", "{}", "duplicate-0"));
        assert.equal(fresh._tag === "accepted" && fresh.outcome, "accepted");
        yield* Queue.take(launches);
        const cleared = yield* service.upsert(
          yield* webhookTaskInput({ schedule: { type: "webhook", dedupe: {} } }),
        );
        assert.deepEqual(
          cleared.task.schedule.type === "webhook" && cleared.task.schedule.dedupe,
          {},
        );
      }),
    ),
);

const waitForSpan = <A, E, R>(work: Effect.Effect<A, E, R>, name: string) =>
  Effect.gen(function* () {
    const done = yield* Deferred.make<void>();
    const tracer = Tracer.make({
      span: (options) => {
        const span = new Tracer.NativeSpan(options);
        const end = span.end.bind(span);
        span.end = (...args) => {
          end(...args);
          if (span.name === name) Deferred.doneUnsafe(done, Effect.void);
        };
        return span;
      },
    });
    const result = yield* work.pipe(Effect.withTracer(tracer));
    yield* Deferred.await(done);
    return result;
  });

const drainPendingDispatch = (workers: Map<string, Effect.Effect<void>>) =>
  waitForSpan(workers.get("webhook-coalescing")!, "ScheduledTaskService.dispatchPendingWebhook");

type LaunchInput = ThreadLaunchService.ThreadLaunchInput;

const webhookTaskInput = (overrides: Record<string, unknown> = {}) =>
  decodeUpsertInput({
    id: "scheduled-task:hook",
    title: "Review PRs",
    prompt: "Review this PR: {{body.pull_request.url}}",
    enabled: true,
    schedule: { type: "webhook" },
    projectId: "project-webhook",
    workspaceStrategy: { type: "root" },
    modelSelection: { instanceId: "codex", model: "gpt-5.4" },
    runtimeMode: "full-access",
    interactionMode: "default",
    ...overrides,
  });

const pullRequestBody = new TextEncoder().encode(
  JSON.stringify({ pull_request: { url: "https://github.com/org/repo/pull/45" } }),
);

const requestFor = (
  task: { readonly id: string; readonly webhook?: { readonly path: string } | undefined },
  overrides: Partial<ScheduledTaskService.WebhookTriggerRequest> = {},
): ScheduledTaskService.WebhookTriggerRequest => ({
  hookId: task.id,
  token: task.webhook?.path.split("/").at(-1) ?? "",
  method: "POST",
  path: `/api/hooks/${task.id}`,
  query: "",
  headers: { "content-type": "application/json" },
  body: pullRequestBody,
  bodyText: new TextDecoder().decode(pullRequestBody),
  ...overrides,
});

const withService = <A, E>(
  body: (input: {
    readonly service: ScheduledTaskService.ScheduledTaskService["Service"];
    readonly launches: Queue.Queue<LaunchInput>;
    readonly sql: SqlClient.SqlClient;
    readonly secretsByRef: Map<string, string>;
  }) => Effect.Effect<A, E, never>,
  options: {
    readonly onFrame?: () => void;
    readonly frameGate?: {
      readonly started: Deferred.Deferred<void>;
      readonly release: Deferred.Deferred<void>;
    };
    readonly completedLaunches?: Set<string>;
    readonly failLaunches?: { remaining: number };
    readonly gate?: Deferred.Deferred<void>;
    readonly gateForTask?: (input: LaunchInput) => Deferred.Deferred<void> | undefined;
    readonly relayHookBaseUrl?: string;
    readonly sql?: SqlClient.SqlClient;
    readonly workers?: Map<string, Effect.Effect<void>>;
    readonly policy?: ManagedPolicyDocument;
  } = {},
) =>
  Effect.gen(function* () {
    const launches = yield* Queue.unbounded<LaunchInput>();
    const secretsByRef = new Map<string, string>();
    const layerDependencies = Layer.mergeAll(
      NodePlatformCrypto.layerDigest,
      NodePlatformCrypto.layerMac,
      options.frameGate === undefined && options.onFrame === undefined
        ? NodePlatformCrypto.layer
        : Layer.effect(
            SecureRandom.SecureRandom,
            Effect.gen(function* () {
              const crypto = yield* SecureRandom.SecureRandom;
              return {
                ...crypto,
                randomBytes: (size: number) =>
                  Effect.suspend(() => {
                    if (size === 6) options.onFrame?.();
                    return size === 6 && options.frameGate !== undefined
                      ? Deferred.succeed(options.frameGate.started, undefined).pipe(
                          Effect.andThen(Deferred.await(options.frameGate.release)),
                          Effect.andThen(crypto.randomBytes(size)),
                        )
                      : crypto.randomBytes(size);
                  }),
              };
            }),
          ).pipe(Layer.provide(NodePlatformCrypto.layer)),
      options.workers === undefined
        ? Scheduler.layer
        : Layer.succeed(Scheduler.Scheduler, {
            register: <E, R>(name: string, work: Effect.Effect<void, E, R>) =>
              Effect.gen(function* () {
                const context = yield* Effect.context<R>();
                options.workers!.set(name, work.pipe(Effect.provideContext(context), Effect.orDie));
              }),
          }),
      Layer.mock(ThreadLaunchService.ThreadLaunchService)({
        launch: (input) =>
          Effect.suspend(() => {
            if (options.completedLaunches?.has(input.commandId))
              return Effect.succeed({ threadId: "thread-1", resumed: false } as never);
            options.completedLaunches?.add(input.commandId);
            return Queue.offer(launches, input).pipe(
              Effect.andThen(
                (options.gateForTask?.(input) ?? options.gate)
                  ? Deferred.await((options.gateForTask?.(input) ?? options.gate)!)
                  : Effect.void,
              ),
              Effect.andThen(
                Effect.sync(() => {
                  if (options.failLaunches && options.failLaunches.remaining-- > 0)
                    throw new Error("injected start failure");
                }),
              ),
              Effect.as({ threadId: "thread-1", resumed: false } as never),
            );
          }),
      }),
      Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
        namedProjectsRoot: "/projects",
        isScratchProject: () => Effect.succeed(false),
      }),
      Layer.mock(ThreadManagementService.ThreadManagementService)({}),
      Layer.mock(SecretRequests.SecretRequests)({
        consume: ({ ref }) => {
          const value = secretsByRef.get(ref);
          secretsByRef.delete(ref);
          return value === undefined
            ? Effect.fail(new SecretRequestError({ reason: "ref_unavailable" }))
            : Effect.succeed(value);
        },
      }),
      Layer.succeed(
        ScheduledTaskService.ScheduledTaskWebhookOrigin,
        Effect.succeed({ relayHookBaseUrl: options.relayHookBaseUrl ?? null }),
      ),
      ManagedPolicy.layerTest(options.policy),
    );
    return yield* Effect.gen(function* () {
      const service = yield* ScheduledTaskService.ScheduledTaskService;
      const sql = yield* SqlClient.SqlClient;
      return yield* body({ service, launches, sql, secretsByRef });
    }).pipe(
      Effect.provide(
        Layer.fresh(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))),
      ),
    );
  }).pipe(
    Effect.provide(
      options.sql === undefined
        ? SqlitePersistence.layerMemory
        : Layer.succeed(SqlClient.SqlClient, options.sql),
    ),
  );

it.effect("dispatches exactly the rendered prompt and logs the delivery", () =>
  withService(({ service, launches }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(yield* webhookTaskInput());
      assert.equal(task.nextRunAt, null);
      assert.isDefined(task.webhook);
      assert.isTrue(task.webhook!.path.startsWith("/api/hooks/scheduled-task%3Ahook/"));
      assert.equal(task.webhook!.url, null);

      const result = yield* service.triggerWebhook(requestFor(task));
      assert.equal(result._tag, "accepted");
      const launched = yield* Queue.take(launches);
      const framed =
        /^Review this PR: <(webhook-data-[0-9a-f]{12})>https:\/\/github\.com\/org\/repo\/pull\/45<\/\1>\n\nText between <\1> and <\/\1> was copied from the webhook request\./;
      const frame = framed.exec(launched.initialMessage?.text ?? "")?.[1];
      assert.isDefined(frame);
      assert.equal(
        launched.commandId,
        `scheduled-task:${task.id}:webhook:${result._tag === "accepted" ? result.deliveryId : ""}`,
      );

      const { deliveries } = yield* service.listWebhookDeliveries({
        id: task.id,
        acceptAnyOutcome: true,
      });
      assert.equal(deliveries.length, 1);
      assert.equal(deliveries[0]?.outcome, "accepted");
      const { delivery } = yield* service.getWebhookDelivery({
        id: task.id,
        deliveryId: deliveries[0]!.id,
      });
      assert.equal(delivery.body, new TextDecoder().decode(pullRequestBody));
      assert.equal(delivery.renderedPrompt, launched.initialMessage?.text);

      yield* service.triggerWebhook(requestFor(task));
      const next = framed.exec((yield* Queue.take(launches)).initialMessage?.text ?? "")?.[1];
      assert.isDefined(next);
      assert.notEqual(next, frame);
    }),
  ),
);

it("builds the relay hook URL from the managed tunnel's key, never the environment id", () => {
  const relayUrl = "https://relay.example.com/";
  assert.equal(
    ScheduledTaskService.relayHookBaseUrl({
      relayUrl,
      tunnelName: "t3coderelay-managedendpoint-dev-julius-0123456789abcdef",
    }),
    "https://relay.example.com/v1/hooks/0123456789abcdef",
  );
  for (const tunnelName of [undefined, "t3coderelay-managedendpoint", "x-0123456789ABCDEF"]) {
    assert.isNull(ScheduledTaskService.relayHookBaseUrl({ relayUrl, tunnelName }));
  }
});

it.effect("gives webhook tasks a relay URL when the environment has a managed tunnel", () =>
  withService(
    ({ service }) =>
      Effect.gen(function* () {
        const { task } = yield* service.upsert(yield* webhookTaskInput());
        const token = task.webhook!.path.split("/").at(-1);
        assert.equal(
          task.webhook?.url,
          `https://relay.example.com/v1/hooks/0123456789abcdef/scheduled-task%3Ahook/${token}`,
        );
      }),
    { relayHookBaseUrl: "https://relay.example.com/v1/hooks/0123456789abcdef" },
  ),
);

it.effect("answers not found for a wrong token or unknown hook without logging", () =>
  withService(({ service }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(yield* webhookTaskInput());
      const wrongToken = yield* service.triggerWebhook(requestFor(task, { token: "nope" }));
      assert.equal(wrongToken._tag, "not_found");
      const unknown = yield* service.triggerWebhook(requestFor(task, { hookId: "missing" }));
      assert.equal(unknown._tag, "not_found");
      assert.equal(
        (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries
          .length,
        0,
      );
    }),
  ),
);

it.effect("rotating the token retires the old URL and saving keeps it", () =>
  withService(({ service }) =>
    Effect.gen(function* () {
      const input = yield* webhookTaskInput();
      const { task } = yield* service.upsert(input);
      const saved = yield* service.upsert(yield* webhookTaskInput({ title: "Renamed" }));
      assert.equal(saved.task.webhook?.path, task.webhook?.path);

      const rotated = yield* service.rotateWebhookToken({ id: task.id });
      assert.notEqual(rotated.task.webhook?.path, task.webhook?.path);
      assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "not_found");
      assert.equal((yield* service.triggerWebhook(requestFor(rotated.task)))._tag, "accepted");
    }),
  ),
);

it.effect("checks the configured signature and keeps the secret write-only", () =>
  withService(({ service }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(
        yield* webhookTaskInput({
          schedule: {
            type: "webhook",
            signature: {
              header: "X-Hub-Signature-256",
              encoding: "hex",
              prefix: "sha256=",
              secret: "s3cret",
            },
          },
        }),
      );
      assert.deepEqual(task.schedule, {
        type: "webhook",
        signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" },
        maxDeliveryAgeMinutes: null,
      });
      assert.isTrue(task.webhook?.hasSecret);

      const unsigned = yield* service.triggerWebhook(requestFor(task));
      assert.equal(unsigned._tag, "rejected_signature");

      const signature = `sha256=${NodeCrypto.createHmac("sha256", "s3cret").update(pullRequestBody).digest("hex")}`;
      const signed = yield* service.triggerWebhook(
        requestFor(task, {
          headers: { "content-type": "application/json", "x-hub-signature-256": signature },
        }),
      );
      assert.equal(signed._tag, "accepted");

      const resaved = yield* service.upsert(
        yield* webhookTaskInput({
          schedule: {
            type: "webhook",
            signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" },
          },
        }),
      );
      assert.isTrue(resaved.task.webhook?.hasSecret);

      const outcomes = (yield* service.listWebhookDeliveries({
        id: task.id,
        acceptAnyOutcome: true,
      })).deliveries.map((delivery) => [delivery.outcome, delivery.signatureVerified]);
      assert.deepEqual(outcomes.toSorted(), [
        ["accepted", true],
        ["rejected_signature", false],
      ]);
    }),
  ),
);

it.effect("a policy without webhook tasks keeps existing ones but refuses saves and requests", () =>
  withService(({ service, sql }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(yield* webhookTaskInput());
      const timer = yield* webhookTaskInput({
        id: "scheduled-task:timer",
        schedule: { type: "interval", everyMs: 3_600_000 },
      });
      yield* withService(
        ({ service, launches }) =>
          Effect.gen(function* () {
            const blocked = "Example Corp's policy does not allow webhook tasks.";
            const listed = (yield* service.list()).tasks.find(({ id }) => id === task.id);
            assert.equal(listed?.blockedByPolicy, blocked);
            assert.isTrue(listed?.enabled);

            const refused = yield* service.upsert(yield* webhookTaskInput()).pipe(Effect.flip);
            assert.equal(refused.message, blocked);
            const created = yield* service
              .upsert(yield* webhookTaskInput({ id: "scheduled-task:new-hook" }))
              .pipe(Effect.flip);
            assert.equal(created.message, blocked);

            // Turned off, it can still be edited, but not turned back on.
            yield* service.setEnabled({ id: task.id, enabled: false });
            yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
            const reenabled = yield* service
              .setEnabled({ id: task.id, enabled: true })
              .pipe(Effect.flip);
            assert.equal(reenabled.message, blocked);

            const result = yield* service.triggerWebhook(requestFor(task));
            assert.equal(result._tag, "blocked_by_policy");
            assert.equal(yield* Queue.size(launches), 0);
            assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_deliveries`).length, 0);

            // Timer tasks are unaffected, including a webhook task switched to a timer.
            assert.isUndefined((yield* service.upsert(timer)).task.blockedByPolicy);
            const switched = yield* service.upsert(
              yield* webhookTaskInput({ schedule: { type: "interval", everyMs: 3_600_000 } }),
            );
            assert.isUndefined(switched.task.blockedByPolicy);
          }),
        {
          sql,
          policy: {
            schemaVersion: 1,
            organization: "Example Corp",
            limits: { webhookTasks: false },
          },
        },
      );
    }),
  ),
);

it.effect("logs but does not run deliveries to a disabled task, and refuses run now", () =>
  withService(({ service, launches }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
      assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "disabled");
      assert.equal(yield* Queue.size(launches), 0);
      const runNow = yield* service.runNow({ id: task.id }).pipe(Effect.flip);
      assert.equal(runNow.message, "Webhook tasks run when their URL receives a request.");
    }),
  ),
);

it.effect("a save carrying a stale token cannot undo a rotation", () =>
  withService(({ service }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(yield* webhookTaskInput());
      const rotated = yield* service.rotateWebhookToken({ id: task.id });
      const saved = yield* service.upsert(yield* webhookTaskInput({ title: "Edited" }));
      assert.equal(saved.task.webhook?.path, rotated.task.webhook?.path);
      assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "not_found");
    }),
  ),
);

it.effect("keeps credential headers and query values out of the delivery log", () =>
  withService(({ service }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
      yield* service.triggerWebhook(
        requestFor(task, {
          query: "page=2&api_key=k",
          headers: {
            "content-type": "application/json",
            authorization: "Bearer sender-token",
            "x-webhook-key": "k",
            "x-github-event": "push",
          },
        }),
      );
      const [summary] = (yield* service.listWebhookDeliveries({
        id: task.id,
        acceptAnyOutcome: true,
      })).deliveries;
      const { delivery } = yield* service.getWebhookDelivery({
        id: task.id,
        deliveryId: summary!.id,
      });
      assert.equal(delivery.query, "page=2&api_key=[redacted]");
      assert.equal(delivery.headers.authorization, "[redacted]");
      assert.equal(delivery.headers["x-webhook-key"], "[redacted]");
      assert.equal(delivery.headers["x-github-event"], "push");
    }),
  ),
);

it.effect("drops a character split by the delivery log's body limit", () =>
  withService(({ service }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
      const limit = 64 * 1024;
      const euro = new TextEncoder().encode("€");
      const body = new Uint8Array(limit - 1 + euro.length).fill(0x61);
      body.set(euro, limit - 1);
      yield* service.triggerWebhook(
        requestFor(task, { body, bodyText: new TextDecoder().decode(body) }),
      );
      const [summary] = (yield* service.listWebhookDeliveries({
        id: task.id,
        acceptAnyOutcome: true,
      })).deliveries;
      const { delivery } = yield* service.getWebhookDelivery({
        id: task.id,
        deliveryId: summary!.id,
      });
      assert.isTrue(delivery.bodyTruncated);
      assert.equal(delivery.bodyBytes, body.byteLength);
      assert.equal(delivery.body.slice(-3), "aaa");
      assert.equal(delivery.body.length, limit - 1);
    }),
  ),
);

const queuedDeliveryCases = [
  {
    change: "paused",
    reason: "The task was paused before this delivery ran.",
    apply: (service: ScheduledTaskService.ScheduledTaskService["Service"], id: string) =>
      service.setEnabled({ id: id as never, enabled: false }),
  },
  {
    change: "switched to an interval trigger",
    reason: "The task's trigger changed before this delivery ran.",
    apply: (service: ScheduledTaskService.ScheduledTaskService["Service"]) =>
      webhookTaskInput({ schedule: { type: "interval", everyMs: 3_600_000 } }).pipe(
        Effect.flatMap(service.upsert),
      ),
  },
] as const;

it.effect.each(queuedDeliveryCases)(
  "a delivery queued behind a run does not start once the task is $change",
  ({ reason, apply }) =>
    Effect.gen(function* () {
      const gate = yield* Deferred.make<void>();
      const deliveryReleased = yield* Deferred.make<void>();
      const tracer = Tracer.make({
        span: (options) => {
          const span = new Tracer.NativeSpan(options);
          const end = span.end.bind(span);
          span.end = (...args) => {
            end(...args);
            if (span.name === "ScheduledTaskService.releaseFailedDelivery") {
              Deferred.doneUnsafe(deliveryReleased, Effect.void);
            }
          };
          return span;
        },
      });
      yield* withService(
        ({ service, launches }) =>
          Effect.gen(function* () {
            const { task } = yield* service.upsert(yield* webhookTaskInput());
            yield* service.triggerWebhook(requestFor(task));
            const queued = yield* service.triggerWebhook(requestFor(task));
            yield* Queue.take(launches);
            yield* apply(service, task.id);
            yield* Deferred.succeed(gate, undefined);
            const deliveryId = queued._tag === "accepted" ? queued.deliveryId : undefined;
            yield* Deferred.await(deliveryReleased);
            const delivery = (yield* service.getWebhookDelivery({
              id: task.id,
              deliveryId: deliveryId!,
            })).delivery;
            assert.equal(delivery.outcome, "dispatch_failed");
            assert.equal(delivery.error, reason);
            assert.equal(yield* Queue.size(launches), 0);
          }),
        { gate },
      ).pipe(Effect.withTracer(tracer));
    }),
);

it.effect("a save without a secret keeps a secret changed after it was read", () =>
  withService(({ service }) =>
    Effect.gen(function* () {
      const signature = { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" };
      const { task } = yield* service.upsert(
        yield* webhookTaskInput({
          schedule: { type: "webhook", signature: { ...signature, secret: "old" } },
        }),
      );
      yield* service.upsert(
        yield* webhookTaskInput({
          schedule: { type: "webhook", signature: { ...signature, secret: "new" } },
        }),
      );
      yield* service.upsert(
        yield* webhookTaskInput({ title: "Edited", schedule: { type: "webhook", signature } }),
      );
      const sign = (secret: string) =>
        `sha256=${NodeCrypto.createHmac("sha256", secret).update(pullRequestBody).digest("hex")}`;
      const withSignature = (secret: string) =>
        requestFor(task, {
          headers: { "content-type": "application/json", "x-hub-signature-256": sign(secret) },
        });
      assert.equal(
        (yield* service.triggerWebhook(withSignature("old")))._tag,
        "rejected_signature",
      );
      assert.equal((yield* service.triggerWebhook(withSignature("new")))._tag, "accepted");
    }),
  ),
);

it.effect("a held request already delivered directly runs only once", () =>
  withService(({ service, launches }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(yield* webhookTaskInput());
      const direct = yield* service.triggerWebhook(
        requestFor(task, { relayDeliveryId: "relay-1" }),
      );
      const replayed = yield* service.triggerWebhook(
        requestFor(task, { relayDeliveryId: "relay-1", receivedAt: "2026-10-04T10:00:00.000Z" }),
      );
      assert.equal(direct._tag, "accepted");
      assert.deepEqual(replayed, { ...direct, outcome: "duplicate" } as typeof replayed);
      yield* Queue.take(launches);
      const logged = (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true }))
        .deliveries;
      assert.equal(logged.length, 1);
    }),
  ),
);

it.effect("relay retries recover failed acceptance without releasing another request's claim", () =>
  withService(({ service, launches, sql }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(yield* webhookTaskInput());
      const request = requestFor(task, { relayDeliveryId: "retry-after-log-failure" });
      yield* sql`
        CREATE TRIGGER fail_webhook_delivery_log
        BEFORE INSERT ON scheduled_task_webhook_deliveries
        BEGIN
          SELECT RAISE(FAIL, 'injected delivery log failure');
        END
      `;
      const error = yield* service.triggerWebhook(request).pipe(Effect.flip);
      assert.equal(error.message, "Could not record webhook delivery.");
      assert.equal(yield* Queue.size(launches), 0);
      assert.isEmpty(
        (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries,
      );
      yield* sql`DROP TRIGGER fail_webhook_delivery_log`;

      const retried = yield* Effect.all(
        [service.triggerWebhook(request), service.triggerWebhook(request)],
        { concurrency: "unbounded" },
      );
      assert.deepEqual(
        retried.map((result) => (result._tag === "accepted" ? result.outcome : result._tag)).sort(),
        ["accepted", "duplicate"],
      );
      assert.include((yield* Queue.take(launches)).commandId, request.relayDeliveryId!);

      yield* sql`
        CREATE TRIGGER fail_webhook_claim
        BEFORE INSERT ON scheduled_task_webhook_relay_deliveries
        BEGIN
          SELECT RAISE(FAIL, 'injected claim failure');
        END
      `;
      const retryError = yield* service.triggerWebhook(request).pipe(Effect.flip);
      assert.equal(retryError.message, "Could not record webhook delivery.");
      yield* sql`DROP TRIGGER fail_webhook_claim`;
      const replayed = yield* service.triggerWebhook(request);
      assert.equal(replayed._tag === "accepted" ? replayed.outcome : replayed._tag, "duplicate");
      assert.equal(yield* Queue.size(launches), 0);
      assert.equal(
        (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries
          .length,
        1,
      );
    }),
  ),
);

it.effect("a held request whose sender hung up mid-request still runs", () =>
  withService(({ service, launches }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(yield* webhookTaskInput());
      for (let step = 0; step < 60; step++) {
        const request = requestFor(task, { relayDeliveryId: `hung-up-${step}` });
        const fiber = yield* service
          .triggerWebhook(request)
          .pipe(Effect.provideService(EffectScheduler.MaxOpsBeforeYield, 8), Effect.forkChild);
        for (let yields = 0; yields < step; yields++) yield* Effect.yieldNow;
        yield* Fiber.interrupt(fiber);
        const retried = yield* service.triggerWebhook(request);
        assert.equal(retried._tag, "accepted");
        const deliveryId = retried._tag === "accepted" ? retried.deliveryId : undefined;
        yield* service.getWebhookDelivery({ id: task.id, deliveryId: deliveryId! });
        const launch = yield* Queue.take(launches);
        assert.include(launch.commandId, `hung-up-${step}`);
        yield* TestClock.adjust("61 seconds");
      }
      assert.equal(yield* Queue.size(launches), 0);
    }),
  ),
);

it.effect("a held request runs once even after the log has trimmed it", () =>
  withService(({ service, launches }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(yield* webhookTaskInput());
      const first = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" }));
      assert.equal(first._tag, "accepted");
      yield* Queue.take(launches);
      const paused = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
      yield* Effect.forEach(Array.from({ length: 55 }), () =>
        service.triggerWebhook(requestFor(paused.task)),
      );
      yield* service.upsert(yield* webhookTaskInput());
      const replay = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" }));
      assert.equal(replay._tag, "accepted");
      assert.equal(yield* Queue.size(launches), 0);
    }),
  ),
);

it.effect("skips a held request older than the task's max age", () =>
  withService(({ service, launches }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(
        yield* webhookTaskInput({ schedule: { type: "webhook", maxDeliveryAgeMinutes: 30 } }),
      );
      const now = yield* DateTime.now;
      const old = DateTime.formatIso(DateTime.subtract(now, { minutes: 31 }));
      const fresh = DateTime.formatIso(DateTime.subtract(now, { minutes: 5 }));
      const tooOld = yield* service.triggerWebhook(
        requestFor(task, { relayDeliveryId: "old", receivedAt: old }),
      );
      assert.equal(tooOld._tag, "expired");
      assert.equal(yield* Queue.size(launches), 0);
      const ok = yield* service.triggerWebhook(
        requestFor(task, { relayDeliveryId: "fresh", receivedAt: fresh }),
      );
      assert.equal(ok._tag, "accepted");
      const outcomes = (yield* service.listWebhookDeliveries({
        id: task.id,
        acceptAnyOutcome: true,
      })).deliveries.map((delivery) => delivery.outcome);
      assert.includeMembers(outcomes, ["expired", "accepted"]);
    }),
  ),
);

const githubSignature = (secret: string) =>
  `sha256=${NodeCrypto.createHmac("sha256", secret).update(pullRequestBody).digest("hex")}`;

it.effect("a signature can take the user's secret by ref, which works only once", () =>
  withService(({ service, launches, secretsByRef }) =>
    Effect.gen(function* () {
      secretsByRef.set("secret-ref:00000000000000000000000000000001", "github-secret");
      const githubSchedule = (secretRef: string) => ({
        type: "webhook",
        signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=", secretRef },
      });
      const { task } = yield* service.upsert(
        yield* webhookTaskInput({
          schedule: githubSchedule("secret-ref:00000000000000000000000000000001"),
        }),
      );
      assert.isTrue(task.webhook!.hasSecret);
      const signed = yield* service.triggerWebhook(
        requestFor(task, {
          headers: {
            "content-type": "application/json",
            "x-hub-signature-256": githubSignature("github-secret"),
          },
        }),
      );
      assert.equal(signed._tag, "accepted");
      yield* Queue.take(launches);

      const reused = yield* service
        .upsert(
          yield* webhookTaskInput({
            id: "scheduled-task:other",
            schedule: githubSchedule("secret-ref:00000000000000000000000000000001"),
          }),
        )
        .pipe(Effect.flip);
      assert.include(reused.message, "already used");
    }),
  ),
);

it.effect("a retried save with an already used secretRef keeps the stored secret", () =>
  withService(({ service, launches, secretsByRef }) =>
    Effect.gen(function* () {
      secretsByRef.set("secret-ref:00000000000000000000000000000002", "github-secret");
      const save = webhookTaskInput({
        id: undefined,
        commandId: "command:mcp:schedule-task:release-hook",
        schedule: {
          type: "webhook",
          signature: {
            header: "x-hub-signature-256",
            encoding: "hex",
            prefix: "sha256=",
            secretRef: "secret-ref:00000000000000000000000000000002",
          },
        },
      });
      const first = yield* service.upsert(yield* save);
      const retried = yield* service.upsert(
        yield* webhookTaskInput({
          id: undefined,
          commandId: "command:mcp:schedule-task:release-hook",
          schedule: {
            type: "webhook",
            signature: {
              header: "x-hub-signature-256",
              encoding: "hex",
              prefix: "sha256=",
              secretRef: "secret-ref:00000000000000000000000000000002",
              secret: "made-up-secret",
            },
          },
        }),
      );
      assert.equal(retried.task.id, first.task.id);
      const signed = yield* service.triggerWebhook(
        requestFor(retried.task, {
          headers: {
            "content-type": "application/json",
            "x-hub-signature-256": githubSignature("github-secret"),
          },
        }),
      );
      assert.equal(signed._tag, "accepted");
      yield* Queue.take(launches);
    }),
  ),
);

it.effect("a signature without any secret is refused", () =>
  withService(({ service }) =>
    Effect.gen(function* () {
      const failure = yield* service
        .upsert(
          yield* webhookTaskInput({
            schedule: {
              type: "webhook",
              signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" },
            },
          }),
        )
        .pipe(Effect.flip);
      assert.include(failure.message, "needs a signing secret");
    }),
  ),
);

it.effect("dedupes replay keys per task, persists them, and expires them after seven days", () =>
  withService(
    ({ service, launches, sql }) =>
      Effect.gen(function* () {
        const schedule = { type: "webhook", dedupe: { replayKey: "{{headers.delivery-id}}" } };
        const { task } = yield* service.upsert(yield* webhookTaskInput({ schedule }));
        const request = requestFor(task, { headers: { "delivery-id": "A" } });
        const first = yield* service.triggerWebhook(request);
        assert.equal(first._tag === "accepted" && first.outcome, "accepted");
        yield* Queue.take(launches);
        const replay = yield* service.triggerWebhook(request);
        assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate");
        assert.equal(yield* Queue.size(launches), 0);
        const other = yield* service.upsert(
          yield* webhookTaskInput({ id: "scheduled-task:other", schedule }),
        );
        yield* service.triggerWebhook(requestFor(other.task, { headers: request.headers }));
        yield* Queue.take(launches);
        yield* withService(
          ({ service: restarted }) =>
            Effect.gen(function* () {
              const replay = yield* restarted.triggerWebhook(request);
              assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate");
            }),
          { sql, workers: new Map() },
        );
        yield* TestClock.adjust("7 days");
        const expired = yield* service.triggerWebhook(request);
        assert.equal(expired._tag === "accepted" && expired.outcome, "accepted");
        yield* Queue.take(launches);
        assert.include(
          (yield* service.listWebhookDeliveries({
            id: task.id,
            acceptAnyOutcome: true,
          })).deliveries.map((d) => d.outcome),
          "duplicate",
        );
      }),
    { workers: new Map() },
  ),
);

it.effect.each([1, 3])(
  "coalesces %i deliveries into one run with the latest body, headers and query",
  (count) => {
    const workers = new Map<string, Effect.Effect<void>>();
    return withService(
      ({ service, launches }) =>
        Effect.gen(function* () {
          const { task } = yield* service.upsert(
            yield* webhookTaskInput({
              prompt: "{{body.value}} {{headers.version}} {{query.version}}",
              schedule: {
                type: "webhook",
                dedupe: { coalesceKey: "{{body.key}}", coalesceWindowSeconds: 10 },
              },
            }),
          );
          for (let value = 1; value <= count; value++) {
            const bodyText = JSON.stringify({ key: "B", value });
            yield* service.triggerWebhook(
              requestFor(task, {
                bodyText,
                body: new TextEncoder().encode(bodyText),
                headers: { version: String(value) },
                query: `version=${value}`,
              }),
            );
            if (value < count) yield* TestClock.adjust("2 seconds");
          }
          const owner = (yield* service.listWebhookDeliveries({
            id: task.id,
            acceptAnyOutcome: true,
          })).deliveries.find((delivery) => delivery.outcome === "accepted")!;
          assert.isTrue(owner.pendingWindow);
          const beforeDispatch = (yield* service.getWebhookDelivery({
            id: task.id,
            deliveryId: owner.id,
            acceptAnyOutcome: true,
          })).delivery;
          assert.isTrue(beforeDispatch.pendingWindow);
          assert.equal(beforeDispatch.body, JSON.stringify({ key: "B", value: count }));
          assert.equal(beforeDispatch.headers.version, String(count));
          assert.equal(beforeDispatch.query, `version=${count}`);
          assert.equal(
            beforeDispatch.bodyBytes,
            new TextEncoder().encode(beforeDispatch.body).byteLength,
          );
          assert.equal(yield* Queue.size(launches), 0);
          yield* workers.get("webhook-coalescing")!;
          assert.equal(yield* Queue.size(launches), 0);
          yield* TestClock.adjust(count === 1 ? "10 seconds" : "6 seconds");
          yield* workers.get("webhook-coalescing")!;
          const launched = yield* Queue.take(launches);
          assert.equal(
            (launched.initialMessage!.text.match(new RegExp(`>${count}<`, "g")) ?? []).length,
            3,
          );
          yield* workers.get("webhook-coalescing")!;
          assert.equal(yield* Queue.size(launches), 0);
          const outcomes = (yield* service.listWebhookDeliveries({
            id: task.id,
            acceptAnyOutcome: true,
          })).deliveries.map((d) => d.outcome);
          const afterDispatch = (yield* service.getWebhookDelivery({
            id: task.id,
            deliveryId: owner.id,
            acceptAnyOutcome: true,
          })).delivery;
          assert.isFalse(afterDispatch.pendingWindow);
          assert.equal(afterDispatch.body, beforeDispatch.body);
          assert.include(afterDispatch.renderedPrompt!, `>${count}<`);
          assert.equal(outcomes.filter((o) => o === "accepted").length, 1);
          assert.equal(outcomes.filter((o) => o === "coalesced").length, count - 1);
        }),
      { workers },
    );
  },
);

it.effect("a coalesced window dispatches the prompt saved after it opened", () => {
  const workers = new Map<string, Effect.Effect<void>>();
  return withService(
    ({ service, launches }) =>
      Effect.gen(function* () {
        const schedule = {
          type: "webhook",
          dedupe: { coalesceKey: "{{body.id}}", coalesceWindowSeconds: 10 },
        };
        const { task } = yield* service.upsert(
          yield* webhookTaskInput({ prompt: "Use the original instructions.", schedule }),
        );
        const bodyText = JSON.stringify({ id: "review" });
        yield* service.triggerWebhook(
          requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }),
        );
        yield* service.upsert(
          yield* webhookTaskInput({ prompt: "Use the updated instructions.", schedule }),
        );
        yield* TestClock.adjust("10 seconds");
        yield* workers.get("webhook-coalescing")!;
        const launched = yield* Queue.take(launches);
        assert.equal(launched.initialMessage?.text, "Use the updated instructions.");
        const owner = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!;
        const { delivery } = yield* service.getWebhookDelivery({
          id: task.id,
          deliveryId: owner.id,
        });
        assert.equal(delivery.renderedPrompt, "Use the updated instructions.");
      }),
    { workers },
  );
});

it.effect("retrying a running window keeps the prompt it was dispatched with", () => {
  const workers = new Map<string, Effect.Effect<void>>();
  return withService(
    ({ service, sql, launches }) =>
      Effect.gen(function* () {
        const schedule = {
          type: "webhook",
          dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
        };
        const save = (prompt: string) =>
          Effect.flatMap(webhookTaskInput({ prompt, schedule }), service.upsert);
        const { task } = yield* save("Prompt when the window opened");
        yield* service.triggerWebhook(requestFor(task));
        yield* save("Prompt at dispatch");
        yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks
          WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`;
        yield* TestClock.adjust("10 seconds");
        yield* drainPendingDispatch(workers);
        const launched = yield* Queue.take(launches);
        assert.equal(launched.initialMessage?.text, "Prompt at dispatch");
        yield* save("Prompt saved after dispatch");
        yield* sql`DROP TRIGGER fail_completion`;
        yield* drainPendingDispatch(workers);
        assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
        const owner = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!;
        const { delivery } = yield* service.getWebhookDelivery({
          id: task.id,
          deliveryId: owner.id,
        });
        assert.equal(delivery.renderedPrompt, "Prompt at dispatch");
      }),
    { workers },
  );
});

it.effect("a coalesced window fails its delivery when the task row cannot be read", () => {
  const workers = new Map<string, Effect.Effect<void>>();
  return withService(
    ({ service, sql, launches }) =>
      Effect.gen(function* () {
        const { task } = yield* service.upsert(
          yield* webhookTaskInput({
            schedule: {
              type: "webhook",
              dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
            },
          }),
        );
        yield* service.triggerWebhook(requestFor(task));
        yield* sql`UPDATE scheduled_tasks SET model_selection_json = 'not json' WHERE task_id = ${task.id}`;
        yield* TestClock.adjust("10 seconds");
        yield* drainPendingDispatch(workers);
        assert.equal(yield* Queue.size(launches), 0);
        assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
        const [delivery] = yield* sql<{
          outcome: string;
          error: string | null;
        }>`SELECT outcome, error FROM scheduled_task_webhook_deliveries`;
        assert.equal(delivery?.outcome, "dispatch_failed");
        // Failing before dispatch means a later successful read cannot send the stale snapshot.
        assert.equal(delivery?.error, "The pending webhook payload could not be prepared.");
      }),
    { workers },
  );
});

it.effect("recovers a persisted coalescing window after restarting the service", () => {
  const workers = new Map<string, Effect.Effect<void>>();
  return withService(
    ({ service, sql }) =>
      Effect.gen(function* () {
        const { task } = yield* service.upsert(
          yield* webhookTaskInput({
            schedule: {
              type: "webhook",
              dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 },
            },
          }),
        );
        yield* service.triggerWebhook(requestFor(task));
        yield* TestClock.adjust("4 seconds");
        const restartedWorkers = new Map<string, Effect.Effect<void>>();
        yield* withService(
          ({ launches }) =>
            Effect.gen(function* () {
              yield* restartedWorkers.get("webhook-coalescing")!;
              assert.equal(yield* Queue.size(launches), 0);
              yield* TestClock.adjust("6 seconds");
              yield* restartedWorkers.get("webhook-coalescing")!;
              yield* Queue.take(launches);
              yield* restartedWorkers.get("webhook-coalescing")!;
              assert.equal(yield* Queue.size(launches), 0);
              assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
            }),
          { sql, workers: restartedWorkers },
        );
      }),
    { workers },
  );
});

it.effect("a pending dispatch interrupted at its permit does not block later polls", () => {
  const workers = new Map<string, Effect.Effect<void>>();
  // One shared dispatcher lets the test run the retry's forked worker to completion.
  const mixed = new EffectScheduler.MixedScheduler("sync");
  const dispatcher = mixed.makeDispatcher();
  return withService(
    ({ service, sql, launches }) =>
      Effect.gen(function* () {
        const { task } = yield* service.upsert(
          yield* webhookTaskInput({
            schedule: {
              type: "webhook",
              dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 },
            },
          }),
        );
        yield* service.triggerWebhook(requestFor(task));
        yield* TestClock.adjust("10 seconds");
        const poll = workers.get("webhook-coalescing")!;

        const entered = yield* Deferred.make<Fiber.Fiber<unknown, unknown>>();
        const interruptAtPermit = Tracer.make({
          span: (options) => {
            if (options.name === "ScheduledTaskService.webhookPermit") {
              const fiber = Fiber.getCurrent()!;
              Deferred.doneUnsafe(entered, Effect.succeed(fiber));
              fiber.interruptUnsafe();
            }
            return new Tracer.NativeSpan(options);
          },
        });
        const firstPoll = yield* poll.pipe(Effect.withTracer(interruptAtPermit), Effect.forkChild);
        assert.isTrue(Exit.hasInterrupts(yield* Fiber.await(yield* Deferred.await(entered))));
        yield* Fiber.await(firstPoll);

        yield* poll;
        dispatcher.flush();

        assert.equal(yield* Queue.size(launches), 1);
        assert.lengthOf(yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`, 0);
      }),
    { workers },
  ).pipe(
    Effect.provideService(EffectScheduler.Scheduler, {
      executionMode: mixed.executionMode,
      makeDispatcher: () => dispatcher,
      shouldYield: (fiber) => mixed.shouldYield(fiber),
    }),
  );
});

it.effect("rolls back replay and pending state when recording a delivery fails", () => {
  const workers = new Map<string, Effect.Effect<void>>();
  return withService(
    ({ service, sql, launches }) =>
      Effect.gen(function* () {
        const { task } = yield* service.upsert(
          yield* webhookTaskInput({
            schedule: {
              type: "webhook",
              dedupe: {
                replayKey: "{{headers.content-type}}",
                coalesceKey: "{{headers.content-type}}",
                coalesceWindowSeconds: 10,
              },
            },
          }),
        );
        yield* sql`CREATE TRIGGER fail_dedupe_log BEFORE INSERT ON scheduled_task_webhook_deliveries BEGIN SELECT RAISE(FAIL, 'injected'); END`;
        yield* service.triggerWebhook(requestFor(task)).pipe(Effect.flip);
        assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
        assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
        yield* sql`DROP TRIGGER fail_dedupe_log`;
        const retry = yield* service.triggerWebhook(requestFor(task));
        assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
        yield* TestClock.adjust("10 seconds");
        yield* workers.get("webhook-coalescing")!;
        yield* Queue.take(launches);
      }),
    { workers },
  );
});

it.effect("queue-full deliveries release replay keys so a later retry runs", () =>
  Effect.gen(function* () {
    const gate = yield* Deferred.make<void>();
    yield* withService(
      ({ service, launches }) =>
        Effect.gen(function* () {
          const { task } = yield* service.upsert(
            yield* webhookTaskInput({
              schedule: { type: "webhook", dedupe: { replayKey: "{{headers.delivery-id}}" } },
            }),
          );
          const request = (i: number) =>
            requestFor(task, { headers: { "delivery-id": String(i) } });
          for (let i = 0; i < 20; i++) yield* service.triggerWebhook(request(i));
          yield* Queue.take(launches);
          const full = yield* service.triggerWebhook(request(20));
          assert.equal(full._tag === "rate_limited" && full.outcome, "queue_full");
          yield* Deferred.succeed(gate, undefined);
          for (let i = 1; i < 20; i++) yield* Queue.take(launches);
          const retry = yield* service.triggerWebhook(request(20));
          assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
          yield* Queue.take(launches);
        }),
      { gate },
    );
  }),
);

it.effect.each(["task_json", "request_json"] as const)(
  "removes corrupt pending %s instead of retrying forever",
  (column) => {
    const workers = new Map<string, Effect.Effect<void>>();
    return withService(
      ({ service, sql }) =>
        Effect.gen(function* () {
          const { task } = yield* service.upsert(
            yield* webhookTaskInput({
              schedule: {
                type: "webhook",
                dedupe: {
                  replayKey: "{{body}}",
                  coalesceKey: "{{body}}",
                  coalesceWindowSeconds: 10,
                },
              },
            }),
          );
          yield* service.triggerWebhook(requestFor(task));
          yield* sql`UPDATE scheduled_task_webhook_pending SET ${sql(column)} = 'broken'`;
          yield* TestClock.adjust("10 seconds");
          yield* drainPendingDispatch(workers);
          assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
          assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
          assert.equal(
            (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!.outcome,
            "dispatch_failed",
          );
          yield* workers.get("webhook-coalescing")!;
        }),
      { workers },
    );
  },
);

it.effect(
  "restarting after a coalesced launch before pending deletion does not launch or count it twice",
  () => {
    const workers = new Map<string, Effect.Effect<void>>();
    return withService(
      ({ service, sql, launches }) =>
        Effect.gen(function* () {
          const { task } = yield* service.upsert(
            yield* webhookTaskInput({
              schedule: {
                type: "webhook",
                dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
              },
            }),
          );
          const accepted = yield* service.triggerWebhook(requestFor(task));
          yield* sql`CREATE TRIGGER fail_pending_delete BEFORE DELETE ON scheduled_task_webhook_pending BEGIN SELECT RAISE(FAIL,'crash after launch'); END`;
          yield* TestClock.adjust("10 seconds");
          yield* drainPendingDispatch(workers);
          const launch = yield* Queue.take(launches);
          assert.equal(
            launch.commandId,
            `scheduled-task:${task.id}:webhook:${accepted._tag === "accepted" && accepted.deliveryId}`,
          );
          assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
          assert.equal(
            (yield* sql`SELECT * FROM scheduled_task_webhook_pending WHERE state = 'completed'`)
              .length,
            1,
          );
          yield* sql`DROP TRIGGER fail_pending_delete`;
          const restartedWorkers = new Map<string, Effect.Effect<void>>();
          yield* withService(
            ({ service: restarted, launches: secondLaunches }) =>
              Effect.gen(function* () {
                yield* drainPendingDispatch(restartedWorkers);
                assert.equal(yield* Queue.size(secondLaunches), 0);
                assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1);
                assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
              }),
            { sql, workers: restartedWorkers },
          );
        }),
      { workers },
    );
  },
);

it.effect("guards bad-signature floods separately from valid run admission", () =>
  withService(({ service, sql }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(
        yield* webhookTaskInput({
          schedule: {
            type: "webhook",
            signature: { header: "signature", encoding: "hex", prefix: "", secret: "secret" },
          },
        }),
      );
      const results = yield* Effect.forEach(Array.from({ length: 75 }), () =>
        service.triggerWebhook(requestFor(task)),
      );
      assert.equal(results.filter((result) => result._tag === "rejected_signature").length, 60);
      assert.equal(results.filter((result) => result._tag === "rate_limited").length, 15);
      assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_deliveries`).length, 51);
    }),
  ),
);

it.effect("start failures release both replay keys and relay claims for redelivery", () =>
  withService(
    ({ service, sql, launches }) =>
      Effect.gen(function* () {
        const { task } = yield* service.upsert(
          yield* webhookTaskInput({
            schedule: { type: "webhook", dedupe: { replayKey: "{{body}}" } },
          }),
        );
        const request = requestFor(task, { relayDeliveryId: "failed-start" });
        yield* waitForSpan(
          service.triggerWebhook(request),
          "ScheduledTaskService.runWebhookDelivery",
        );
        yield* Queue.take(launches);
        assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
        assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_relay_deliveries`).length, 0);
        assert.equal((yield* service.list()).tasks[0]!.lastRunError, "injected start failure");
        const retry = yield* waitForSpan(
          service.triggerWebhook(request),
          "ScheduledTaskService.runWebhookDelivery",
        );
        assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
        yield* Queue.take(launches);
        assert.equal((yield* service.list()).tasks[0]!.lastRunStatus, "succeeded");
      }),
    { failLaunches: { remaining: 1 } },
  ),
);

it.effect("a start failure releases the relay claim when the task row is replaced mid-run", () =>
  Effect.gen(function* () {
    const gate = yield* Deferred.make<void>();
    yield* withService(
      ({ service, sql, launches }) =>
        Effect.gen(function* () {
          const { task } = yield* service.upsert(yield* webhookTaskInput());
          const request = requestFor(task, { relayDeliveryId: "replaced-mid-run" });
          const first = yield* waitForSpan(
            service.triggerWebhook(request),
            "ScheduledTaskService.runWebhookDelivery",
          ).pipe(Effect.forkChild);
          yield* Queue.take(launches);
          // A same-id recreate while the launch is out: the stored row is no longer this run's.
          yield* sql`UPDATE scheduled_tasks SET last_run_status = 'never', last_run_at = NULL`;
          yield* Deferred.succeed(gate, undefined);
          yield* Fiber.join(first);
          const retry = yield* service.triggerWebhook(request);
          assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
          yield* Queue.take(launches);
        }),
      { gate, failLaunches: { remaining: 1 } },
    );
  }),
);

it.effect("main retention keeps fifty runs and preserves filtered and dedupe buckets", () =>
  withService(({ service, sql, launches }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(yield* webhookTaskInput());
      yield* service.triggerWebhook(requestFor(task));
      yield* Queue.take(launches);
      for (const outcome of ["filtered", "duplicate"]) {
        yield* sql`WITH RECURSIVE n(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM n WHERE value < 20)
        INSERT INTO scheduled_task_webhook_deliveries
        SELECT ${outcome} || n.value, d.task_id, d.received_at, d.method, d.query, d.headers_json, d.body,
          d.body_bytes, d.body_truncated, ${outcome}, d.signature_verified, d.missing_fields_json, d.rendered_prompt, d.error
        FROM n CROSS JOIN (SELECT * FROM scheduled_task_webhook_deliveries WHERE outcome = 'accepted' LIMIT 1) d`;
      }
      for (let i = 0; i < 59; i++) {
        assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "accepted");
        yield* Queue.take(launches);
      }
      const counts = yield* sql<{
        outcome: string;
        count: number;
      }>`SELECT outcome, COUNT(*) AS count FROM scheduled_task_webhook_deliveries GROUP BY outcome ORDER BY outcome`;
      assert.deepEqual(counts, [
        { outcome: "accepted", count: 50 },
        { outcome: "duplicate", count: 20 },
        { outcome: "filtered", count: 20 },
      ]);
    }),
  ),
);

it.effect(
  "a valid request refunds its own reservation without expiring an interleaved rejection",
  () =>
    Effect.gen(function* () {
      const started = yield* Deferred.make<void>();
      const release = yield* Deferred.make<void>();
      yield* withService(
        ({ service, launches }) =>
          Effect.gen(function* () {
            const { task } = yield* service.upsert(
              yield* webhookTaskInput({
                schedule: {
                  type: "webhook",
                  signature: { header: "signature", encoding: "hex", prefix: "", secret: "secret" },
                },
              }),
            );
            const signature = NodeCrypto.createHmac("sha256", "secret")
              .update(pullRequestBody)
              .digest("hex");
            const valid = yield* service
              .triggerWebhook(requestFor(task, { headers: { signature } }))
              .pipe(Effect.forkChild);
            yield* Deferred.await(started);
            yield* TestClock.adjust("30 seconds");
            assert.equal(
              (yield* service.triggerWebhook(requestFor(task)))._tag,
              "rejected_signature",
            );
            yield* Deferred.succeed(release, undefined);
            assert.equal((yield* Fiber.join(valid))._tag, "accepted");
            yield* Queue.take(launches);
            // Only the valid reservation would have expired now; the rejection must still count.
            yield* TestClock.adjust("31 seconds");
            for (let i = 0; i < 59; i++)
              assert.equal(
                (yield* service.triggerWebhook(requestFor(task)))._tag,
                "rejected_signature",
              );
            assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "rate_limited");
          }),
        { frameGate: { started, release } },
      );
    }),
);

it.effect("keeps replay keys when dispatch succeeds but completion bookkeeping fails", () =>
  withService(({ service, sql, launches }) =>
    Effect.gen(function* () {
      const { task } = yield* service.upsert(
        yield* webhookTaskInput({
          schedule: {
            type: "webhook",
            dedupe: { replayKey: "{{body}}" },
          },
        }),
      );
      yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks
      WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`;
      yield* waitForSpan(
        service.triggerWebhook(requestFor(task)),
        "ScheduledTaskService.runWebhookDelivery",
      );
      yield* Queue.take(launches);
      assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 1);
      yield* sql`DROP TRIGGER fail_completion`;
      const retry = yield* service.triggerWebhook(requestFor(task));
      assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate");
      assert.equal(yield* Queue.size(launches), 0);
      assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
    }),
  ),
);

it.effect.each([119_000, 120_000])(
  "bounds coalesced prompt logs and checks trimmed length for %i body characters",
  (length) => {
    const workers = new Map<string, Effect.Effect<void>>();
    return withService(
      ({ service, launches }) =>
        Effect.gen(function* () {
          const { task } = yield* service.upsert(
            yield* webhookTaskInput({
              prompt: `{{body.absent}}${" ".repeat(10_000)}{{body.text}}`,
              schedule: {
                type: "webhook",
                dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 },
              },
            }),
          );
          const bodyText = JSON.stringify({ group: "B", text: "x".repeat(length) });
          const accepted = yield* service.triggerWebhook(
            requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }),
          );
          assert.equal(accepted._tag, "accepted");
          yield* TestClock.adjust("10 seconds");
          yield* drainPendingDispatch(workers);
          const delivery = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!;
          if (length === 119_000) {
            const launch = yield* Queue.take(launches);
            assert.isAbove(launch.initialMessage!.text.length, 120_000);
            assert.equal(delivery.outcome, "accepted");
            const detail = yield* service.getWebhookDelivery({
              id: task.id,
              deliveryId: delivery.id,
            });
            assert.equal(detail.delivery.renderedPrompt!.length, 64 * 1024);
          } else {
            assert.equal(yield* Queue.size(launches), 0);
            assert.equal(delivery.outcome, "dispatch_failed");
            assert.equal(delivery.error, "The filled-in prompt is too long.");
          }
        }),
      { workers },