// @effect-diagnostics-next-line nodeBuiltinImport:off -- Effect's Crypto has no createHmac.
import * as NodeCrypto from "node:crypto";
import * as NodePlatformCrypto from "@effect/platform-node/NodeCrypto";
import { assert, it } from "@effect/vitest";
import {
type ManagedPolicyDocument,
ScheduledTaskUpsertInput,
ScheduledTaskWebhookDeliveryId,
SecretRequestError,
} from "@t3tools/contracts";
import * as DateTime from "effect/DateTime";
import * as SecureRandom from "effect/crypto/SecureRandom";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as Fiber from "effect/Fiber";
import * as Layer from "effect/Layer";
import * as Queue from "effect/Queue";
import * as EffectScheduler from "effect/Scheduler";
import * as Schema from "effect/Schema";
import * as SqlClient from "effect/sql/SqlClient";
import * as TestClock from "effect/testing/TestClock";
import * as Tracer from "effect/Tracer";
import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
import * as ManagedPolicy from "../policy/ManagedPolicy.ts";
import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
import * as SecretRequests from "../secrets/SecretRequests.ts";
import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts";
import * as SqlitePersistence from "../persistence/Sqlite.ts";
import * as Scheduler from "../scheduling/Scheduler.ts";
import * as ScheduledTaskService from "./ScheduledTaskService.ts";
import { renderWebhookPrompt } from "./webhookTemplate.ts";
const decodeUpsertInput = Schema.decodeUnknownEffect(ScheduledTaskUpsertInput);
it.effect(
"preserves omitted dedupe settings, releases duplicate claims, and retains coalesced claims",
() =>
withService(({ service, sql, launches }) =>
Effect.gen(function* () {
const dedupe = {
replayKey: "{{headers.delivery-id}}",
coalesceKey: "{{body.group}}",
coalesceWindowSeconds: 10,
};
const { task } = yield* service.upsert(
yield* webhookTaskInput({ schedule: { type: "webhook", dedupe } }),
);
const updated = yield* service.upsert(yield* webhookTaskInput());
assert.deepEqual(
updated.task.schedule.type === "webhook" && updated.task.schedule.dedupe,
dedupe,
);
const request = (id: string, bodyText = "{}", relayDeliveryId?: string) =>
requestFor(task, {
headers: { "delivery-id": id },
bodyText,
body: new TextEncoder().encode(bodyText),
...(relayDeliveryId === undefined ? {} : { relayDeliveryId }),
});
yield* service.triggerWebhook(request("A"));
yield* Queue.take(launches);
yield* service.triggerWebhook(request("B1", '{"group":"B"}'));
for (let i = 0; i < 5; i++) {
const duplicate = yield* service.triggerWebhook(request("A", "{}", `duplicate-${i}`));
assert.equal(duplicate._tag === "accepted" && duplicate.outcome, "duplicate");
const coalesced = yield* service.triggerWebhook(
request(`B${i + 2}`, '{"group":"B"}', `coalesced-${i}`),
);
assert.equal(coalesced._tag === "accepted" && coalesced.outcome, "coalesced");
}
const claims = yield* sql<{
relay_delivery_id: string;
}>`SELECT relay_delivery_id FROM scheduled_task_webhook_relay_deliveries ORDER BY relay_delivery_id`;
assert.deepEqual(
claims.map((claim) => claim.relay_delivery_id),
Array.from({ length: 5 }, (_, i) => `coalesced-${i}`),
);
assert.isTrue(
(yield* service.listWebhookDeliveries({ id: task.id })).deliveries.every(
(delivery) => delivery.outcome !== "duplicate" && delivery.outcome !== "coalesced",
),
);
const fresh = yield* service.triggerWebhook(request("C", "{}", "duplicate-0"));
assert.equal(fresh._tag === "accepted" && fresh.outcome, "accepted");
yield* Queue.take(launches);
const cleared = yield* service.upsert(
yield* webhookTaskInput({ schedule: { type: "webhook", dedupe: {} } }),
);
assert.deepEqual(
cleared.task.schedule.type === "webhook" && cleared.task.schedule.dedupe,
{},
);
}),
),
);
const waitForSpan = <A, E, R>(work: Effect.Effect<A, E, R>, name: string) =>
Effect.gen(function* () {
const done = yield* Deferred.make<void>();
const tracer = Tracer.make({
span: (options) => {
const span = new Tracer.NativeSpan(options);
const end = span.end.bind(span);
span.end = (...args) => {
end(...args);
if (span.name === name) Deferred.doneUnsafe(done, Effect.void);
};
return span;
},
});
const result = yield* work.pipe(Effect.withTracer(tracer));
yield* Deferred.await(done);
return result;
});
const drainPendingDispatch = (workers: Map<string, Effect.Effect<void>>) =>
waitForSpan(workers.get("webhook-coalescing")!, "ScheduledTaskService.dispatchPendingWebhook");
type LaunchInput = ThreadLaunchService.ThreadLaunchInput;
const webhookTaskInput = (overrides: Record<string, unknown> = {}) =>
decodeUpsertInput({
id: "scheduled-task:hook",
title: "Review PRs",
prompt: "Review this PR: {{body.pull_request.url}}",
enabled: true,
schedule: { type: "webhook" },
projectId: "project-webhook",
workspaceStrategy: { type: "root" },
modelSelection: { instanceId: "codex", model: "gpt-5.4" },
runtimeMode: "full-access",
interactionMode: "default",
...overrides,
});
const pullRequestBody = new TextEncoder().encode(
JSON.stringify({ pull_request: { url: "https://github.com/org/repo/pull/45" } }),
);
const requestFor = (
task: { readonly id: string; readonly webhook?: { readonly path: string } | undefined },
overrides: Partial<ScheduledTaskService.WebhookTriggerRequest> = {},
): ScheduledTaskService.WebhookTriggerRequest => ({
hookId: task.id,
token: task.webhook?.path.split("/").at(-1) ?? "",
method: "POST",
path: `/api/hooks/${task.id}`,
query: "",
headers: { "content-type": "application/json" },
body: pullRequestBody,
bodyText: new TextDecoder().decode(pullRequestBody),
...overrides,
});
const withService = <A, E>(
body: (input: {
readonly service: ScheduledTaskService.ScheduledTaskService["Service"];
readonly launches: Queue.Queue<LaunchInput>;
readonly sql: SqlClient.SqlClient;
readonly secretsByRef: Map<string, string>;
}) => Effect.Effect<A, E, never>,
options: {
readonly onFrame?: () => void;
readonly frameGate?: {
readonly started: Deferred.Deferred<void>;
readonly release: Deferred.Deferred<void>;
};
readonly completedLaunches?: Set<string>;
readonly failLaunches?: { remaining: number };
readonly gate?: Deferred.Deferred<void>;
readonly gateForTask?: (input: LaunchInput) => Deferred.Deferred<void> | undefined;
readonly relayHookBaseUrl?: string;
readonly sql?: SqlClient.SqlClient;
readonly workers?: Map<string, Effect.Effect<void>>;
readonly policy?: ManagedPolicyDocument;
} = {},
) =>
Effect.gen(function* () {
const launches = yield* Queue.unbounded<LaunchInput>();
const secretsByRef = new Map<string, string>();
const layerDependencies = Layer.mergeAll(
NodePlatformCrypto.layerDigest,
NodePlatformCrypto.layerMac,
options.frameGate === undefined && options.onFrame === undefined
? NodePlatformCrypto.layer
: Layer.effect(
SecureRandom.SecureRandom,
Effect.gen(function* () {
const crypto = yield* SecureRandom.SecureRandom;
return {
...crypto,
randomBytes: (size: number) =>
Effect.suspend(() => {
if (size === 6) options.onFrame?.();
return size === 6 && options.frameGate !== undefined
? Deferred.succeed(options.frameGate.started, undefined).pipe(
Effect.andThen(Deferred.await(options.frameGate.release)),
Effect.andThen(crypto.randomBytes(size)),
)
: crypto.randomBytes(size);
}),
};
}),
).pipe(Layer.provide(NodePlatformCrypto.layer)),
options.workers === undefined
? Scheduler.layer
: Layer.succeed(Scheduler.Scheduler, {
register: <E, R>(name: string, work: Effect.Effect<void, E, R>) =>
Effect.gen(function* () {
const context = yield* Effect.context<R>();
options.workers!.set(name, work.pipe(Effect.provideContext(context), Effect.orDie));
}),
}),
Layer.mock(ThreadLaunchService.ThreadLaunchService)({
launch: (input) =>
Effect.suspend(() => {
if (options.completedLaunches?.has(input.commandId))
return Effect.succeed({ threadId: "thread-1", resumed: false } as never);
options.completedLaunches?.add(input.commandId);
return Queue.offer(launches, input).pipe(
Effect.andThen(
(options.gateForTask?.(input) ?? options.gate)
? Deferred.await((options.gateForTask?.(input) ?? options.gate)!)
: Effect.void,
),
Effect.andThen(
Effect.sync(() => {
if (options.failLaunches && options.failLaunches.remaining-- > 0)
throw new Error("injected start failure");
}),
),
Effect.as({ threadId: "thread-1", resumed: false } as never),
);
}),
}),
Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
namedProjectsRoot: "/projects",
isScratchProject: () => Effect.succeed(false),
}),
Layer.mock(ThreadManagementService.ThreadManagementService)({}),
Layer.mock(SecretRequests.SecretRequests)({
consume: ({ ref }) => {
const value = secretsByRef.get(ref);
secretsByRef.delete(ref);
return value === undefined
? Effect.fail(new SecretRequestError({ reason: "ref_unavailable" }))
: Effect.succeed(value);
},
}),
Layer.succeed(
ScheduledTaskService.ScheduledTaskWebhookOrigin,
Effect.succeed({ relayHookBaseUrl: options.relayHookBaseUrl ?? null }),
),
ManagedPolicy.layerTest(options.policy),
);
return yield* Effect.gen(function* () {
const service = yield* ScheduledTaskService.ScheduledTaskService;
const sql = yield* SqlClient.SqlClient;
return yield* body({ service, launches, sql, secretsByRef });
}).pipe(
Effect.provide(
Layer.fresh(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))),
),
);
}).pipe(
Effect.provide(
options.sql === undefined
? SqlitePersistence.layerMemory
: Layer.succeed(SqlClient.SqlClient, options.sql),
),
);
it.effect("dispatches exactly the rendered prompt and logs the delivery", () =>
withService(({ service, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput());
assert.equal(task.nextRunAt, null);
assert.isDefined(task.webhook);
assert.isTrue(task.webhook!.path.startsWith("/api/hooks/scheduled-task%3Ahook/"));
assert.equal(task.webhook!.url, null);
const result = yield* service.triggerWebhook(requestFor(task));
assert.equal(result._tag, "accepted");
const launched = yield* Queue.take(launches);
const framed =
/^Review this PR: <(webhook-data-[0-9a-f]{12})>https:\/\/github\.com\/org\/repo\/pull\/45<\/\1>\n\nText between <\1> and <\/\1> was copied from the webhook request\./;
const frame = framed.exec(launched.initialMessage?.text ?? "")?.[1];
assert.isDefined(frame);
assert.equal(
launched.commandId,
`scheduled-task:${task.id}:webhook:${result._tag === "accepted" ? result.deliveryId : ""}`,
);
const { deliveries } = yield* service.listWebhookDeliveries({
id: task.id,
acceptAnyOutcome: true,
});
assert.equal(deliveries.length, 1);
assert.equal(deliveries[0]?.outcome, "accepted");
const { delivery } = yield* service.getWebhookDelivery({
id: task.id,
deliveryId: deliveries[0]!.id,
});
assert.equal(delivery.body, new TextDecoder().decode(pullRequestBody));
assert.equal(delivery.renderedPrompt, launched.initialMessage?.text);
yield* service.triggerWebhook(requestFor(task));
const next = framed.exec((yield* Queue.take(launches)).initialMessage?.text ?? "")?.[1];
assert.isDefined(next);
assert.notEqual(next, frame);
}),
),
);
it("builds the relay hook URL from the managed tunnel's key, never the environment id", () => {
const relayUrl = "https://relay.example.com/";
assert.equal(
ScheduledTaskService.relayHookBaseUrl({
relayUrl,
tunnelName: "t3coderelay-managedendpoint-dev-julius-0123456789abcdef",
}),
"https://relay.example.com/v1/hooks/0123456789abcdef",
);
for (const tunnelName of [undefined, "t3coderelay-managedendpoint", "x-0123456789ABCDEF"]) {
assert.isNull(ScheduledTaskService.relayHookBaseUrl({ relayUrl, tunnelName }));
}
});
it.effect("gives webhook tasks a relay URL when the environment has a managed tunnel", () =>
withService(
({ service }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput());
const token = task.webhook!.path.split("/").at(-1);
assert.equal(
task.webhook?.url,
`https://relay.example.com/v1/hooks/0123456789abcdef/scheduled-task%3Ahook/${token}`,
);
}),
{ relayHookBaseUrl: "https://relay.example.com/v1/hooks/0123456789abcdef" },
),
);
it.effect("answers not found for a wrong token or unknown hook without logging", () =>
withService(({ service }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput());
const wrongToken = yield* service.triggerWebhook(requestFor(task, { token: "nope" }));
assert.equal(wrongToken._tag, "not_found");
const unknown = yield* service.triggerWebhook(requestFor(task, { hookId: "missing" }));
assert.equal(unknown._tag, "not_found");
assert.equal(
(yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries
.length,
0,
);
}),
),
);
it.effect("rotating the token retires the old URL and saving keeps it", () =>
withService(({ service }) =>
Effect.gen(function* () {
const input = yield* webhookTaskInput();
const { task } = yield* service.upsert(input);
const saved = yield* service.upsert(yield* webhookTaskInput({ title: "Renamed" }));
assert.equal(saved.task.webhook?.path, task.webhook?.path);
const rotated = yield* service.rotateWebhookToken({ id: task.id });
assert.notEqual(rotated.task.webhook?.path, task.webhook?.path);
assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "not_found");
assert.equal((yield* service.triggerWebhook(requestFor(rotated.task)))._tag, "accepted");
}),
),
);
it.effect("checks the configured signature and keeps the secret write-only", () =>
withService(({ service }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({
schedule: {
type: "webhook",
signature: {
header: "X-Hub-Signature-256",
encoding: "hex",
prefix: "sha256=",
secret: "s3cret",
},
},
}),
);
assert.deepEqual(task.schedule, {
type: "webhook",
signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" },
maxDeliveryAgeMinutes: null,
});
assert.isTrue(task.webhook?.hasSecret);
const unsigned = yield* service.triggerWebhook(requestFor(task));
assert.equal(unsigned._tag, "rejected_signature");
const signature = `sha256=${NodeCrypto.createHmac("sha256", "s3cret").update(pullRequestBody).digest("hex")}`;
const signed = yield* service.triggerWebhook(
requestFor(task, {
headers: { "content-type": "application/json", "x-hub-signature-256": signature },
}),
);
assert.equal(signed._tag, "accepted");
const resaved = yield* service.upsert(
yield* webhookTaskInput({
schedule: {
type: "webhook",
signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" },
},
}),
);
assert.isTrue(resaved.task.webhook?.hasSecret);
const outcomes = (yield* service.listWebhookDeliveries({
id: task.id,
acceptAnyOutcome: true,
})).deliveries.map((delivery) => [delivery.outcome, delivery.signatureVerified]);
assert.deepEqual(outcomes.toSorted(), [
["accepted", true],
["rejected_signature", false],
]);
}),
),
);
it.effect("a policy without webhook tasks keeps existing ones but refuses saves and requests", () =>
withService(({ service, sql }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput());
const timer = yield* webhookTaskInput({
id: "scheduled-task:timer",
schedule: { type: "interval", everyMs: 3_600_000 },
});
yield* withService(
({ service, launches }) =>
Effect.gen(function* () {
const blocked = "Example Corp's policy does not allow webhook tasks.";
const listed = (yield* service.list()).tasks.find(({ id }) => id === task.id);
assert.equal(listed?.blockedByPolicy, blocked);
assert.isTrue(listed?.enabled);
const refused = yield* service.upsert(yield* webhookTaskInput()).pipe(Effect.flip);
assert.equal(refused.message, blocked);
const created = yield* service
.upsert(yield* webhookTaskInput({ id: "scheduled-task:new-hook" }))
.pipe(Effect.flip);
assert.equal(created.message, blocked);
// Turned off, it can still be edited, but not turned back on.
yield* service.setEnabled({ id: task.id, enabled: false });
yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
const reenabled = yield* service
.setEnabled({ id: task.id, enabled: true })
.pipe(Effect.flip);
assert.equal(reenabled.message, blocked);
const result = yield* service.triggerWebhook(requestFor(task));
assert.equal(result._tag, "blocked_by_policy");
assert.equal(yield* Queue.size(launches), 0);
assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_deliveries`).length, 0);
// Timer tasks are unaffected, including a webhook task switched to a timer.
assert.isUndefined((yield* service.upsert(timer)).task.blockedByPolicy);
const switched = yield* service.upsert(
yield* webhookTaskInput({ schedule: { type: "interval", everyMs: 3_600_000 } }),
);
assert.isUndefined(switched.task.blockedByPolicy);
}),
{
sql,
policy: {
schemaVersion: 1,
organization: "Example Corp",
limits: { webhookTasks: false },
},
},
);
}),
),
);
it.effect("logs but does not run deliveries to a disabled task, and refuses run now", () =>
withService(({ service, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "disabled");
assert.equal(yield* Queue.size(launches), 0);
const runNow = yield* service.runNow({ id: task.id }).pipe(Effect.flip);
assert.equal(runNow.message, "Webhook tasks run when their URL receives a request.");
}),
),
);
it.effect("a save carrying a stale token cannot undo a rotation", () =>
withService(({ service }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput());
const rotated = yield* service.rotateWebhookToken({ id: task.id });
const saved = yield* service.upsert(yield* webhookTaskInput({ title: "Edited" }));
assert.equal(saved.task.webhook?.path, rotated.task.webhook?.path);
assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "not_found");
}),
),
);
it.effect("keeps credential headers and query values out of the delivery log", () =>
withService(({ service }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
yield* service.triggerWebhook(
requestFor(task, {
query: "page=2&api_key=k",
headers: {
"content-type": "application/json",
authorization: "Bearer sender-token",
"x-webhook-key": "k",
"x-github-event": "push",
},
}),
);
const [summary] = (yield* service.listWebhookDeliveries({
id: task.id,
acceptAnyOutcome: true,
})).deliveries;
const { delivery } = yield* service.getWebhookDelivery({
id: task.id,
deliveryId: summary!.id,
});
assert.equal(delivery.query, "page=2&api_key=[redacted]");
assert.equal(delivery.headers.authorization, "[redacted]");
assert.equal(delivery.headers["x-webhook-key"], "[redacted]");
assert.equal(delivery.headers["x-github-event"], "push");
}),
),
);
it.effect("drops a character split by the delivery log's body limit", () =>
withService(({ service }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
const limit = 64 * 1024;
const euro = new TextEncoder().encode("€");
const body = new Uint8Array(limit - 1 + euro.length).fill(0x61);
body.set(euro, limit - 1);
yield* service.triggerWebhook(
requestFor(task, { body, bodyText: new TextDecoder().decode(body) }),
);
const [summary] = (yield* service.listWebhookDeliveries({
id: task.id,
acceptAnyOutcome: true,
})).deliveries;
const { delivery } = yield* service.getWebhookDelivery({
id: task.id,
deliveryId: summary!.id,
});
assert.isTrue(delivery.bodyTruncated);
assert.equal(delivery.bodyBytes, body.byteLength);
assert.equal(delivery.body.slice(-3), "aaa");
assert.equal(delivery.body.length, limit - 1);
}),
),
);
const queuedDeliveryCases = [
{
change: "paused",
reason: "The task was paused before this delivery ran.",
apply: (service: ScheduledTaskService.ScheduledTaskService["Service"], id: string) =>
service.setEnabled({ id: id as never, enabled: false }),
},
{
change: "switched to an interval trigger",
reason: "The task's trigger changed before this delivery ran.",
apply: (service: ScheduledTaskService.ScheduledTaskService["Service"]) =>
webhookTaskInput({ schedule: { type: "interval", everyMs: 3_600_000 } }).pipe(
Effect.flatMap(service.upsert),
),
},
] as const;
it.effect.each(queuedDeliveryCases)(
"a delivery queued behind a run does not start once the task is $change",
({ reason, apply }) =>
Effect.gen(function* () {
const gate = yield* Deferred.make<void>();
const deliveryReleased = yield* Deferred.make<void>();
const tracer = Tracer.make({
span: (options) => {
const span = new Tracer.NativeSpan(options);
const end = span.end.bind(span);
span.end = (...args) => {
end(...args);
if (span.name === "ScheduledTaskService.releaseFailedDelivery") {
Deferred.doneUnsafe(deliveryReleased, Effect.void);
}
};
return span;
},
});
yield* withService(
({ service, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput());
yield* service.triggerWebhook(requestFor(task));
const queued = yield* service.triggerWebhook(requestFor(task));
yield* Queue.take(launches);
yield* apply(service, task.id);
yield* Deferred.succeed(gate, undefined);
const deliveryId = queued._tag === "accepted" ? queued.deliveryId : undefined;
yield* Deferred.await(deliveryReleased);
const delivery = (yield* service.getWebhookDelivery({
id: task.id,
deliveryId: deliveryId!,
})).delivery;
assert.equal(delivery.outcome, "dispatch_failed");
assert.equal(delivery.error, reason);
assert.equal(yield* Queue.size(launches), 0);
}),
{ gate },
).pipe(Effect.withTracer(tracer));
}),
);
it.effect("a save without a secret keeps a secret changed after it was read", () =>
withService(({ service }) =>
Effect.gen(function* () {
const signature = { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" };
const { task } = yield* service.upsert(
yield* webhookTaskInput({
schedule: { type: "webhook", signature: { ...signature, secret: "old" } },
}),
);
yield* service.upsert(
yield* webhookTaskInput({
schedule: { type: "webhook", signature: { ...signature, secret: "new" } },
}),
);
yield* service.upsert(
yield* webhookTaskInput({ title: "Edited", schedule: { type: "webhook", signature } }),
);
const sign = (secret: string) =>
`sha256=${NodeCrypto.createHmac("sha256", secret).update(pullRequestBody).digest("hex")}`;
const withSignature = (secret: string) =>
requestFor(task, {
headers: { "content-type": "application/json", "x-hub-signature-256": sign(secret) },
});
assert.equal(
(yield* service.triggerWebhook(withSignature("old")))._tag,
"rejected_signature",
);
assert.equal((yield* service.triggerWebhook(withSignature("new")))._tag, "accepted");
}),
),
);
it.effect("a held request already delivered directly runs only once", () =>
withService(({ service, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput());
const direct = yield* service.triggerWebhook(
requestFor(task, { relayDeliveryId: "relay-1" }),
);
const replayed = yield* service.triggerWebhook(
requestFor(task, { relayDeliveryId: "relay-1", receivedAt: "2026-10-04T10:00:00.000Z" }),
);
assert.equal(direct._tag, "accepted");
assert.deepEqual(replayed, { ...direct, outcome: "duplicate" } as typeof replayed);
yield* Queue.take(launches);
const logged = (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true }))
.deliveries;
assert.equal(logged.length, 1);
}),
),
);
it.effect("relay retries recover failed acceptance without releasing another request's claim", () =>
withService(({ service, launches, sql }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput());
const request = requestFor(task, { relayDeliveryId: "retry-after-log-failure" });
yield* sql`
CREATE TRIGGER fail_webhook_delivery_log
BEFORE INSERT ON scheduled_task_webhook_deliveries
BEGIN
SELECT RAISE(FAIL, 'injected delivery log failure');
END
`;
const error = yield* service.triggerWebhook(request).pipe(Effect.flip);
assert.equal(error.message, "Could not record webhook delivery.");
assert.equal(yield* Queue.size(launches), 0);
assert.isEmpty(
(yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries,
);
yield* sql`DROP TRIGGER fail_webhook_delivery_log`;
const retried = yield* Effect.all(
[service.triggerWebhook(request), service.triggerWebhook(request)],
{ concurrency: "unbounded" },
);
assert.deepEqual(
retried.map((result) => (result._tag === "accepted" ? result.outcome : result._tag)).sort(),
["accepted", "duplicate"],
);
assert.include((yield* Queue.take(launches)).commandId, request.relayDeliveryId!);
yield* sql`
CREATE TRIGGER fail_webhook_claim
BEFORE INSERT ON scheduled_task_webhook_relay_deliveries
BEGIN
SELECT RAISE(FAIL, 'injected claim failure');
END
`;
const retryError = yield* service.triggerWebhook(request).pipe(Effect.flip);
assert.equal(retryError.message, "Could not record webhook delivery.");
yield* sql`DROP TRIGGER fail_webhook_claim`;
const replayed = yield* service.triggerWebhook(request);
assert.equal(replayed._tag === "accepted" ? replayed.outcome : replayed._tag, "duplicate");
assert.equal(yield* Queue.size(launches), 0);
assert.equal(
(yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries
.length,
1,
);
}),
),
);
it.effect("a held request whose sender hung up mid-request still runs", () =>
withService(({ service, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput());
for (let step = 0; step < 60; step++) {
const request = requestFor(task, { relayDeliveryId: `hung-up-${step}` });
const fiber = yield* service
.triggerWebhook(request)
.pipe(Effect.provideService(EffectScheduler.MaxOpsBeforeYield, 8), Effect.forkChild);
for (let yields = 0; yields < step; yields++) yield* Effect.yieldNow;
yield* Fiber.interrupt(fiber);
const retried = yield* service.triggerWebhook(request);
assert.equal(retried._tag, "accepted");
const deliveryId = retried._tag === "accepted" ? retried.deliveryId : undefined;
yield* service.getWebhookDelivery({ id: task.id, deliveryId: deliveryId! });
const launch = yield* Queue.take(launches);
assert.include(launch.commandId, `hung-up-${step}`);
yield* TestClock.adjust("61 seconds");
}
assert.equal(yield* Queue.size(launches), 0);
}),
),
);
it.effect("a held request runs once even after the log has trimmed it", () =>
withService(({ service, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput());
const first = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" }));
assert.equal(first._tag, "accepted");
yield* Queue.take(launches);
const paused = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
yield* Effect.forEach(Array.from({ length: 55 }), () =>
service.triggerWebhook(requestFor(paused.task)),
);
yield* service.upsert(yield* webhookTaskInput());
const replay = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" }));
assert.equal(replay._tag, "accepted");
assert.equal(yield* Queue.size(launches), 0);
}),
),
);
it.effect("skips a held request older than the task's max age", () =>
withService(({ service, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({ schedule: { type: "webhook", maxDeliveryAgeMinutes: 30 } }),
);
const now = yield* DateTime.now;
const old = DateTime.formatIso(DateTime.subtract(now, { minutes: 31 }));
const fresh = DateTime.formatIso(DateTime.subtract(now, { minutes: 5 }));
const tooOld = yield* service.triggerWebhook(
requestFor(task, { relayDeliveryId: "old", receivedAt: old }),
);
assert.equal(tooOld._tag, "expired");
assert.equal(yield* Queue.size(launches), 0);
const ok = yield* service.triggerWebhook(
requestFor(task, { relayDeliveryId: "fresh", receivedAt: fresh }),
);
assert.equal(ok._tag, "accepted");
const outcomes = (yield* service.listWebhookDeliveries({
id: task.id,
acceptAnyOutcome: true,
})).deliveries.map((delivery) => delivery.outcome);
assert.includeMembers(outcomes, ["expired", "accepted"]);
}),
),
);
const githubSignature = (secret: string) =>
`sha256=${NodeCrypto.createHmac("sha256", secret).update(pullRequestBody).digest("hex")}`;
it.effect("a signature can take the user's secret by ref, which works only once", () =>
withService(({ service, launches, secretsByRef }) =>
Effect.gen(function* () {
secretsByRef.set("secret-ref:00000000000000000000000000000001", "github-secret");
const githubSchedule = (secretRef: string) => ({
type: "webhook",
signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=", secretRef },
});
const { task } = yield* service.upsert(
yield* webhookTaskInput({
schedule: githubSchedule("secret-ref:00000000000000000000000000000001"),
}),
);
assert.isTrue(task.webhook!.hasSecret);
const signed = yield* service.triggerWebhook(
requestFor(task, {
headers: {
"content-type": "application/json",
"x-hub-signature-256": githubSignature("github-secret"),
},
}),
);
assert.equal(signed._tag, "accepted");
yield* Queue.take(launches);
const reused = yield* service
.upsert(
yield* webhookTaskInput({
id: "scheduled-task:other",
schedule: githubSchedule("secret-ref:00000000000000000000000000000001"),
}),
)
.pipe(Effect.flip);
assert.include(reused.message, "already used");
}),
),
);
it.effect("a retried save with an already used secretRef keeps the stored secret", () =>
withService(({ service, launches, secretsByRef }) =>
Effect.gen(function* () {
secretsByRef.set("secret-ref:00000000000000000000000000000002", "github-secret");
const save = webhookTaskInput({
id: undefined,
commandId: "command:mcp:schedule-task:release-hook",
schedule: {
type: "webhook",
signature: {
header: "x-hub-signature-256",
encoding: "hex",
prefix: "sha256=",
secretRef: "secret-ref:00000000000000000000000000000002",
},
},
});
const first = yield* service.upsert(yield* save);
const retried = yield* service.upsert(
yield* webhookTaskInput({
id: undefined,
commandId: "command:mcp:schedule-task:release-hook",
schedule: {
type: "webhook",
signature: {
header: "x-hub-signature-256",
encoding: "hex",
prefix: "sha256=",
secretRef: "secret-ref:00000000000000000000000000000002",
secret: "made-up-secret",
},
},
}),
);
assert.equal(retried.task.id, first.task.id);
const signed = yield* service.triggerWebhook(
requestFor(retried.task, {
headers: {
"content-type": "application/json",
"x-hub-signature-256": githubSignature("github-secret"),
},
}),
);
assert.equal(signed._tag, "accepted");
yield* Queue.take(launches);
}),
),
);
it.effect("a signature without any secret is refused", () =>
withService(({ service }) =>
Effect.gen(function* () {
const failure = yield* service
.upsert(
yield* webhookTaskInput({
schedule: {
type: "webhook",
signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" },
},
}),
)
.pipe(Effect.flip);
assert.include(failure.message, "needs a signing secret");
}),
),
);
it.effect("dedupes replay keys per task, persists them, and expires them after seven days", () =>
withService(
({ service, launches, sql }) =>
Effect.gen(function* () {
const schedule = { type: "webhook", dedupe: { replayKey: "{{headers.delivery-id}}" } };
const { task } = yield* service.upsert(yield* webhookTaskInput({ schedule }));
const request = requestFor(task, { headers: { "delivery-id": "A" } });
const first = yield* service.triggerWebhook(request);
assert.equal(first._tag === "accepted" && first.outcome, "accepted");
yield* Queue.take(launches);
const replay = yield* service.triggerWebhook(request);
assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate");
assert.equal(yield* Queue.size(launches), 0);
const other = yield* service.upsert(
yield* webhookTaskInput({ id: "scheduled-task:other", schedule }),
);
yield* service.triggerWebhook(requestFor(other.task, { headers: request.headers }));
yield* Queue.take(launches);
yield* withService(
({ service: restarted }) =>
Effect.gen(function* () {
const replay = yield* restarted.triggerWebhook(request);
assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate");
}),
{ sql, workers: new Map() },
);
yield* TestClock.adjust("7 days");
const expired = yield* service.triggerWebhook(request);
assert.equal(expired._tag === "accepted" && expired.outcome, "accepted");
yield* Queue.take(launches);
assert.include(
(yield* service.listWebhookDeliveries({
id: task.id,
acceptAnyOutcome: true,
})).deliveries.map((d) => d.outcome),
"duplicate",
);
}),
{ workers: new Map() },
),
);
it.effect.each([1, 3])(
"coalesces %i deliveries into one run with the latest body, headers and query",
(count) => {
const workers = new Map<string, Effect.Effect<void>>();
return withService(
({ service, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({
prompt: "{{body.value}} {{headers.version}} {{query.version}}",
schedule: {
type: "webhook",
dedupe: { coalesceKey: "{{body.key}}", coalesceWindowSeconds: 10 },
},
}),
);
for (let value = 1; value <= count; value++) {
const bodyText = JSON.stringify({ key: "B", value });
yield* service.triggerWebhook(
requestFor(task, {
bodyText,
body: new TextEncoder().encode(bodyText),
headers: { version: String(value) },
query: `version=${value}`,
}),
);
if (value < count) yield* TestClock.adjust("2 seconds");
}
const owner = (yield* service.listWebhookDeliveries({
id: task.id,
acceptAnyOutcome: true,
})).deliveries.find((delivery) => delivery.outcome === "accepted")!;
assert.isTrue(owner.pendingWindow);
const beforeDispatch = (yield* service.getWebhookDelivery({
id: task.id,
deliveryId: owner.id,
acceptAnyOutcome: true,
})).delivery;
assert.isTrue(beforeDispatch.pendingWindow);
assert.equal(beforeDispatch.body, JSON.stringify({ key: "B", value: count }));
assert.equal(beforeDispatch.headers.version, String(count));
assert.equal(beforeDispatch.query, `version=${count}`);
assert.equal(
beforeDispatch.bodyBytes,
new TextEncoder().encode(beforeDispatch.body).byteLength,
);
assert.equal(yield* Queue.size(launches), 0);
yield* workers.get("webhook-coalescing")!;
assert.equal(yield* Queue.size(launches), 0);
yield* TestClock.adjust(count === 1 ? "10 seconds" : "6 seconds");
yield* workers.get("webhook-coalescing")!;
const launched = yield* Queue.take(launches);
assert.equal(
(launched.initialMessage!.text.match(new RegExp(`>${count}<`, "g")) ?? []).length,
3,
);
yield* workers.get("webhook-coalescing")!;
assert.equal(yield* Queue.size(launches), 0);
const outcomes = (yield* service.listWebhookDeliveries({
id: task.id,
acceptAnyOutcome: true,
})).deliveries.map((d) => d.outcome);
const afterDispatch = (yield* service.getWebhookDelivery({
id: task.id,
deliveryId: owner.id,
acceptAnyOutcome: true,
})).delivery;
assert.isFalse(afterDispatch.pendingWindow);
assert.equal(afterDispatch.body, beforeDispatch.body);
assert.include(afterDispatch.renderedPrompt!, `>${count}<`);
assert.equal(outcomes.filter((o) => o === "accepted").length, 1);
assert.equal(outcomes.filter((o) => o === "coalesced").length, count - 1);
}),
{ workers },
);
},
);
it.effect("a coalesced window dispatches the prompt saved after it opened", () => {
const workers = new Map<string, Effect.Effect<void>>();
return withService(
({ service, launches }) =>
Effect.gen(function* () {
const schedule = {
type: "webhook",
dedupe: { coalesceKey: "{{body.id}}", coalesceWindowSeconds: 10 },
};
const { task } = yield* service.upsert(
yield* webhookTaskInput({ prompt: "Use the original instructions.", schedule }),
);
const bodyText = JSON.stringify({ id: "review" });
yield* service.triggerWebhook(
requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }),
);
yield* service.upsert(
yield* webhookTaskInput({ prompt: "Use the updated instructions.", schedule }),
);
yield* TestClock.adjust("10 seconds");
yield* workers.get("webhook-coalescing")!;
const launched = yield* Queue.take(launches);
assert.equal(launched.initialMessage?.text, "Use the updated instructions.");
const owner = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!;
const { delivery } = yield* service.getWebhookDelivery({
id: task.id,
deliveryId: owner.id,
});
assert.equal(delivery.renderedPrompt, "Use the updated instructions.");
}),
{ workers },
);
});
it.effect("retrying a running window keeps the prompt it was dispatched with", () => {
const workers = new Map<string, Effect.Effect<void>>();
return withService(
({ service, sql, launches }) =>
Effect.gen(function* () {
const schedule = {
type: "webhook",
dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
};
const save = (prompt: string) =>
Effect.flatMap(webhookTaskInput({ prompt, schedule }), service.upsert);
const { task } = yield* save("Prompt when the window opened");
yield* service.triggerWebhook(requestFor(task));
yield* save("Prompt at dispatch");
yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks
WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`;
yield* TestClock.adjust("10 seconds");
yield* drainPendingDispatch(workers);
const launched = yield* Queue.take(launches);
assert.equal(launched.initialMessage?.text, "Prompt at dispatch");
yield* save("Prompt saved after dispatch");
yield* sql`DROP TRIGGER fail_completion`;
yield* drainPendingDispatch(workers);
assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
const owner = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!;
const { delivery } = yield* service.getWebhookDelivery({
id: task.id,
deliveryId: owner.id,
});
assert.equal(delivery.renderedPrompt, "Prompt at dispatch");
}),
{ workers },
);
});
it.effect("a coalesced window fails its delivery when the task row cannot be read", () => {
const workers = new Map<string, Effect.Effect<void>>();
return withService(
({ service, sql, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({
schedule: {
type: "webhook",
dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
},
}),
);
yield* service.triggerWebhook(requestFor(task));
yield* sql`UPDATE scheduled_tasks SET model_selection_json = 'not json' WHERE task_id = ${task.id}`;
yield* TestClock.adjust("10 seconds");
yield* drainPendingDispatch(workers);
assert.equal(yield* Queue.size(launches), 0);
assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
const [delivery] = yield* sql<{
outcome: string;
error: string | null;
}>`SELECT outcome, error FROM scheduled_task_webhook_deliveries`;
assert.equal(delivery?.outcome, "dispatch_failed");
// Failing before dispatch means a later successful read cannot send the stale snapshot.
assert.equal(delivery?.error, "The pending webhook payload could not be prepared.");
}),
{ workers },
);
});
it.effect("recovers a persisted coalescing window after restarting the service", () => {
const workers = new Map<string, Effect.Effect<void>>();
return withService(
({ service, sql }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({
schedule: {
type: "webhook",
dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 },
},
}),
);
yield* service.triggerWebhook(requestFor(task));
yield* TestClock.adjust("4 seconds");
const restartedWorkers = new Map<string, Effect.Effect<void>>();
yield* withService(
({ launches }) =>
Effect.gen(function* () {
yield* restartedWorkers.get("webhook-coalescing")!;
assert.equal(yield* Queue.size(launches), 0);
yield* TestClock.adjust("6 seconds");
yield* restartedWorkers.get("webhook-coalescing")!;
yield* Queue.take(launches);
yield* restartedWorkers.get("webhook-coalescing")!;
assert.equal(yield* Queue.size(launches), 0);
assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
}),
{ sql, workers: restartedWorkers },
);
}),
{ workers },
);
});
it.effect("a pending dispatch interrupted at its permit does not block later polls", () => {
const workers = new Map<string, Effect.Effect<void>>();
// One shared dispatcher lets the test run the retry's forked worker to completion.
const mixed = new EffectScheduler.MixedScheduler("sync");
const dispatcher = mixed.makeDispatcher();
return withService(
({ service, sql, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({
schedule: {
type: "webhook",
dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 },
},
}),
);
yield* service.triggerWebhook(requestFor(task));
yield* TestClock.adjust("10 seconds");
const poll = workers.get("webhook-coalescing")!;
const entered = yield* Deferred.make<Fiber.Fiber<unknown, unknown>>();
const interruptAtPermit = Tracer.make({
span: (options) => {
if (options.name === "ScheduledTaskService.webhookPermit") {
const fiber = Fiber.getCurrent()!;
Deferred.doneUnsafe(entered, Effect.succeed(fiber));
fiber.interruptUnsafe();
}
return new Tracer.NativeSpan(options);
},
});
const firstPoll = yield* poll.pipe(Effect.withTracer(interruptAtPermit), Effect.forkChild);
assert.isTrue(Exit.hasInterrupts(yield* Fiber.await(yield* Deferred.await(entered))));
yield* Fiber.await(firstPoll);
yield* poll;
dispatcher.flush();
assert.equal(yield* Queue.size(launches), 1);
assert.lengthOf(yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`, 0);
}),
{ workers },
).pipe(
Effect.provideService(EffectScheduler.Scheduler, {
executionMode: mixed.executionMode,
makeDispatcher: () => dispatcher,
shouldYield: (fiber) => mixed.shouldYield(fiber),
}),
);
});
it.effect("rolls back replay and pending state when recording a delivery fails", () => {
const workers = new Map<string, Effect.Effect<void>>();
return withService(
({ service, sql, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({
schedule: {
type: "webhook",
dedupe: {
replayKey: "{{headers.content-type}}",
coalesceKey: "{{headers.content-type}}",
coalesceWindowSeconds: 10,
},
},
}),
);
yield* sql`CREATE TRIGGER fail_dedupe_log BEFORE INSERT ON scheduled_task_webhook_deliveries BEGIN SELECT RAISE(FAIL, 'injected'); END`;
yield* service.triggerWebhook(requestFor(task)).pipe(Effect.flip);
assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
yield* sql`DROP TRIGGER fail_dedupe_log`;
const retry = yield* service.triggerWebhook(requestFor(task));
assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
yield* TestClock.adjust("10 seconds");
yield* workers.get("webhook-coalescing")!;
yield* Queue.take(launches);
}),
{ workers },
);
});
it.effect("queue-full deliveries release replay keys so a later retry runs", () =>
Effect.gen(function* () {
const gate = yield* Deferred.make<void>();
yield* withService(
({ service, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({
schedule: { type: "webhook", dedupe: { replayKey: "{{headers.delivery-id}}" } },
}),
);
const request = (i: number) =>
requestFor(task, { headers: { "delivery-id": String(i) } });
for (let i = 0; i < 20; i++) yield* service.triggerWebhook(request(i));
yield* Queue.take(launches);
const full = yield* service.triggerWebhook(request(20));
assert.equal(full._tag === "rate_limited" && full.outcome, "queue_full");
yield* Deferred.succeed(gate, undefined);
for (let i = 1; i < 20; i++) yield* Queue.take(launches);
const retry = yield* service.triggerWebhook(request(20));
assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
yield* Queue.take(launches);
}),
{ gate },
);
}),
);
it.effect.each(["task_json", "request_json"] as const)(
"removes corrupt pending %s instead of retrying forever",
(column) => {
const workers = new Map<string, Effect.Effect<void>>();
return withService(
({ service, sql }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({
schedule: {
type: "webhook",
dedupe: {
replayKey: "{{body}}",
coalesceKey: "{{body}}",
coalesceWindowSeconds: 10,
},
},
}),
);
yield* service.triggerWebhook(requestFor(task));
yield* sql`UPDATE scheduled_task_webhook_pending SET ${sql(column)} = 'broken'`;
yield* TestClock.adjust("10 seconds");
yield* drainPendingDispatch(workers);
assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
assert.equal(
(yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!.outcome,
"dispatch_failed",
);
yield* workers.get("webhook-coalescing")!;
}),
{ workers },
);
},
);
it.effect(
"restarting after a coalesced launch before pending deletion does not launch or count it twice",
() => {
const workers = new Map<string, Effect.Effect<void>>();
return withService(
({ service, sql, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({
schedule: {
type: "webhook",
dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
},
}),
);
const accepted = yield* service.triggerWebhook(requestFor(task));
yield* sql`CREATE TRIGGER fail_pending_delete BEFORE DELETE ON scheduled_task_webhook_pending BEGIN SELECT RAISE(FAIL,'crash after launch'); END`;
yield* TestClock.adjust("10 seconds");
yield* drainPendingDispatch(workers);
const launch = yield* Queue.take(launches);
assert.equal(
launch.commandId,
`scheduled-task:${task.id}:webhook:${accepted._tag === "accepted" && accepted.deliveryId}`,
);
assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
assert.equal(
(yield* sql`SELECT * FROM scheduled_task_webhook_pending WHERE state = 'completed'`)
.length,
1,
);
yield* sql`DROP TRIGGER fail_pending_delete`;
const restartedWorkers = new Map<string, Effect.Effect<void>>();
yield* withService(
({ service: restarted, launches: secondLaunches }) =>
Effect.gen(function* () {
yield* drainPendingDispatch(restartedWorkers);
assert.equal(yield* Queue.size(secondLaunches), 0);
assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1);
assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
}),
{ sql, workers: restartedWorkers },
);
}),
{ workers },
);
},
);
it.effect("guards bad-signature floods separately from valid run admission", () =>
withService(({ service, sql }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({
schedule: {
type: "webhook",
signature: { header: "signature", encoding: "hex", prefix: "", secret: "secret" },
},
}),
);
const results = yield* Effect.forEach(Array.from({ length: 75 }), () =>
service.triggerWebhook(requestFor(task)),
);
assert.equal(results.filter((result) => result._tag === "rejected_signature").length, 60);
assert.equal(results.filter((result) => result._tag === "rate_limited").length, 15);
assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_deliveries`).length, 51);
}),
),
);
it.effect("start failures release both replay keys and relay claims for redelivery", () =>
withService(
({ service, sql, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({
schedule: { type: "webhook", dedupe: { replayKey: "{{body}}" } },
}),
);
const request = requestFor(task, { relayDeliveryId: "failed-start" });
yield* waitForSpan(
service.triggerWebhook(request),
"ScheduledTaskService.runWebhookDelivery",
);
yield* Queue.take(launches);
assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_relay_deliveries`).length, 0);
assert.equal((yield* service.list()).tasks[0]!.lastRunError, "injected start failure");
const retry = yield* waitForSpan(
service.triggerWebhook(request),
"ScheduledTaskService.runWebhookDelivery",
);
assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
yield* Queue.take(launches);
assert.equal((yield* service.list()).tasks[0]!.lastRunStatus, "succeeded");
}),
{ failLaunches: { remaining: 1 } },
),
);
it.effect("a start failure releases the relay claim when the task row is replaced mid-run", () =>
Effect.gen(function* () {
const gate = yield* Deferred.make<void>();
yield* withService(
({ service, sql, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput());
const request = requestFor(task, { relayDeliveryId: "replaced-mid-run" });
const first = yield* waitForSpan(
service.triggerWebhook(request),
"ScheduledTaskService.runWebhookDelivery",
).pipe(Effect.forkChild);
yield* Queue.take(launches);
// A same-id recreate while the launch is out: the stored row is no longer this run's.
yield* sql`UPDATE scheduled_tasks SET last_run_status = 'never', last_run_at = NULL`;
yield* Deferred.succeed(gate, undefined);
yield* Fiber.join(first);
const retry = yield* service.triggerWebhook(request);
assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
yield* Queue.take(launches);
}),
{ gate, failLaunches: { remaining: 1 } },
);
}),
);
it.effect("main retention keeps fifty runs and preserves filtered and dedupe buckets", () =>
withService(({ service, sql, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(yield* webhookTaskInput());
yield* service.triggerWebhook(requestFor(task));
yield* Queue.take(launches);
for (const outcome of ["filtered", "duplicate"]) {
yield* sql`WITH RECURSIVE n(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM n WHERE value < 20)
INSERT INTO scheduled_task_webhook_deliveries
SELECT ${outcome} || n.value, d.task_id, d.received_at, d.method, d.query, d.headers_json, d.body,
d.body_bytes, d.body_truncated, ${outcome}, d.signature_verified, d.missing_fields_json, d.rendered_prompt, d.error
FROM n CROSS JOIN (SELECT * FROM scheduled_task_webhook_deliveries WHERE outcome = 'accepted' LIMIT 1) d`;
}
for (let i = 0; i < 59; i++) {
assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "accepted");
yield* Queue.take(launches);
}
const counts = yield* sql<{
outcome: string;
count: number;
}>`SELECT outcome, COUNT(*) AS count FROM scheduled_task_webhook_deliveries GROUP BY outcome ORDER BY outcome`;
assert.deepEqual(counts, [
{ outcome: "accepted", count: 50 },
{ outcome: "duplicate", count: 20 },
{ outcome: "filtered", count: 20 },
]);
}),
),
);
it.effect(
"a valid request refunds its own reservation without expiring an interleaved rejection",
() =>
Effect.gen(function* () {
const started = yield* Deferred.make<void>();
const release = yield* Deferred.make<void>();
yield* withService(
({ service, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({
schedule: {
type: "webhook",
signature: { header: "signature", encoding: "hex", prefix: "", secret: "secret" },
},
}),
);
const signature = NodeCrypto.createHmac("sha256", "secret")
.update(pullRequestBody)
.digest("hex");
const valid = yield* service
.triggerWebhook(requestFor(task, { headers: { signature } }))
.pipe(Effect.forkChild);
yield* Deferred.await(started);
yield* TestClock.adjust("30 seconds");
assert.equal(
(yield* service.triggerWebhook(requestFor(task)))._tag,
"rejected_signature",
);
yield* Deferred.succeed(release, undefined);
assert.equal((yield* Fiber.join(valid))._tag, "accepted");
yield* Queue.take(launches);
// Only the valid reservation would have expired now; the rejection must still count.
yield* TestClock.adjust("31 seconds");
for (let i = 0; i < 59; i++)
assert.equal(
(yield* service.triggerWebhook(requestFor(task)))._tag,
"rejected_signature",
);
assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "rate_limited");
}),
{ frameGate: { started, release } },
);
}),
);
it.effect("keeps replay keys when dispatch succeeds but completion bookkeeping fails", () =>
withService(({ service, sql, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({
schedule: {
type: "webhook",
dedupe: { replayKey: "{{body}}" },
},
}),
);
yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks
WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`;
yield* waitForSpan(
service.triggerWebhook(requestFor(task)),
"ScheduledTaskService.runWebhookDelivery",
);
yield* Queue.take(launches);
assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 1);
yield* sql`DROP TRIGGER fail_completion`;
const retry = yield* service.triggerWebhook(requestFor(task));
assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate");
assert.equal(yield* Queue.size(launches), 0);
assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
}),
),
);
it.effect.each([119_000, 120_000])(
"bounds coalesced prompt logs and checks trimmed length for %i body characters",
(length) => {
const workers = new Map<string, Effect.Effect<void>>();
return withService(
({ service, launches }) =>
Effect.gen(function* () {
const { task } = yield* service.upsert(
yield* webhookTaskInput({
prompt: `{{body.absent}}${" ".repeat(10_000)}{{body.text}}`,
schedule: {
type: "webhook",
dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 },
},
}),
);
const bodyText = JSON.stringify({ group: "B", text: "x".repeat(length) });
const accepted = yield* service.triggerWebhook(
requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }),
);
assert.equal(accepted._tag, "accepted");
yield* TestClock.adjust("10 seconds");
yield* drainPendingDispatch(workers);
const delivery = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!;
if (length === 119_000) {
const launch = yield* Queue.take(launches);
assert.isAbove(launch.initialMessage!.text.length, 120_000);
assert.equal(delivery.outcome, "accepted");
const detail = yield* service.getWebhookDelivery({
id: task.id,
deliveryId: delivery.id,
});
assert.equal(detail.delivery.renderedPrompt!.length, 64 * 1024);
} else {
assert.equal(yield* Queue.size(launches), 0);
assert.equal(delivery.outcome, "dispatch_failed");
assert.equal(delivery.error, "The filled-in prompt is too long.");
}
}),
{ workers },