import * as Clock from "effect/Clock";
import * as DateTime from "effect/DateTime";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Option from "effect/Option";
import * as Pull from "effect/Pull";
import * as Schedule from "effect/Schedule";
import * as Schema from "effect/Schema";
import * as SqlClient from "effect/sql/SqlClient";
const HOOK_INBOX_TTL_MS = 24 * 60 * 60 * 1000;
const HOOK_INBOX_MAX_REQUESTS = 1_000;
export const HOOK_INBOX_MAX_BYTES = 50 * 1_048_576;
// The relay cannot check tokens; isolate each hook's junk with its own cap.
export const HOOK_INBOX_MAX_PER_HOOK = 100;
const DELIVERIES_PER_RUN = 20;
const ROWS_READ_PER_PAGE = 50;
const BUSY_RETRY_MS = 30_000;
const MAX_RETRY_DELAY = Duration.minutes(10);
const MAX_COUNTED_FAILURES = 25;
/** Cloudflare can take minutes to route a new tunnel; retry every 10 s for three minutes before backing off. */
const retrySchedule = Schedule.spaced("10 seconds").pipe(
Schedule.upTo({ times: 18 }),
Schedule.concat(
Schedule.exponential("30 seconds").pipe(
Schedule.modifyDelay(({ duration }) =>
Effect.succeed(Duration.min(Duration.fromInputUnsafe(duration), MAX_RETRY_DELAY)),
),
),
),
);
export interface HeldHook {
readonly id: string;
readonly receivedAt: string;
readonly method: string;
/** Path segments exactly as the sender sent them; the environment decodes them. */
readonly rawHookId: string;
readonly rawToken: string;
/** The decoded hook id, so every spelling of one hook shares its cap. */
readonly hookKey: string;
readonly query: string;
readonly headers: Readonly<Record<string, string>>;
readonly body: Uint8Array;
}
export type DeliveryOutcome = "delivered" | "busy" | "unreachable";
const retryDelayMs = Effect.fn("HookInboxStore.retryDelayMs")(function* (failures: number) {
const step = yield* Schedule.toStep(retrySchedule);
let delay = MAX_RETRY_DELAY;
for (
let attempt = 0;
attempt < Math.min(Math.max(1, failures), MAX_COUNTED_FAILURES);
attempt++
) {
const next = yield* step(0, undefined).pipe(
Effect.map(([, duration]) => Option.some(duration)),
Pull.catchDone(() => Effect.succeedNone),
);
if (Option.isNone(next)) break;
delay = next.value;
}
return Duration.toMillis(delay);
});
const HeadersJson = Schema.fromJsonString(Schema.Record(Schema.String, Schema.String));
const encodeHeaders = Schema.encodeSync(HeadersJson);
const decodeHeaders = Schema.decodeUnknownOption(HeadersJson);
interface HeldHookRow {
readonly seq: number;
readonly id: string;
readonly received_at: string;
readonly method: string;
readonly raw_hook_id: string;
readonly raw_token: string;
readonly hook_key: string;
readonly query: string;
readonly headers: string;
readonly body: Uint8Array;
}
interface TargetRow {
readonly base_url: string;
readonly failures: number;
}
const iso = (epochMillis: number) => DateTime.formatIso(DateTime.makeUnsafe(epochMillis));
export const migrate = Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
yield* sql`
CREATE TABLE IF NOT EXISTS held_hooks (
seq INTEGER PRIMARY KEY AUTOINCREMENT,
id TEXT NOT NULL UNIQUE,
received_at TEXT NOT NULL,
method TEXT NOT NULL,
raw_hook_id TEXT NOT NULL,
raw_token TEXT NOT NULL,
hook_key TEXT NOT NULL,
query TEXT NOT NULL,
headers TEXT NOT NULL,
body BLOB NOT NULL
)
`;
yield* sql`CREATE INDEX IF NOT EXISTS held_hooks_hook ON held_hooks (hook_key)`;
yield* sql`
CREATE TABLE IF NOT EXISTS held_hooks_target (
id INTEGER PRIMARY KEY CHECK (id = 1),
base_url TEXT NOT NULL,
failures INTEGER NOT NULL DEFAULT 0
)
`;
});
const setTarget = Effect.fn("HookInboxStore.setTarget")(function* (
baseUrl: string,
options: { readonly resetFailures: boolean },
) {
const sql = yield* SqlClient.SqlClient;
yield* options.resetFailures
? sql`
INSERT INTO held_hooks_target (id, base_url) VALUES (1, ${baseUrl})
ON CONFLICT (id) DO UPDATE SET base_url = excluded.base_url, failures = 0
`
: sql`
INSERT INTO held_hooks_target (id, base_url) VALUES (1, ${baseUrl})
ON CONFLICT (id) DO UPDATE SET base_url = excluded.base_url
`;
});
const readTarget = Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
const rows = yield* sql<TargetRow>`SELECT base_url, failures FROM held_hooks_target WHERE id = 1`;
return rows[0] ?? null;
});
const resetFailures = Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
yield* sql`UPDATE held_hooks_target SET failures = 0 WHERE id = 1`;
});
// Increment in place to preserve wake resets made during an in-flight request.
const countFailure = Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
const rows = yield* sql<{ readonly failures: number }>`
UPDATE held_hooks_target SET failures = min(failures + 1, ${MAX_COUNTED_FAILURES})
WHERE id = 1 RETURNING failures
`;
return rows[0]?.failures ?? 1;
});
const hasPending = Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
const rows = yield* sql<{ readonly id: string }>`SELECT id FROM held_hooks LIMIT 1`;
return rows.length > 0;
});
/** First attempt's due time, or null if the full inbox stored nothing. */
export const hold = Effect.fn("HookInboxStore.hold")(function* (hook: HeldHook, baseUrl: string) {
const sql = yield* SqlClient.SqlClient;
// One statement enforces caps across concurrent requests.
const inserted = yield* sql<{ readonly id: string }>`
INSERT INTO held_hooks
(id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body)
SELECT ${hook.id}, ${hook.receivedAt}, ${hook.method}, ${hook.rawHookId}, ${hook.rawToken},
${hook.hookKey}, ${hook.query}, ${encodeHeaders(hook.headers)}, ${hook.body}
WHERE (SELECT count(*) FROM held_hooks) < ${HOOK_INBOX_MAX_REQUESTS}
AND (SELECT count(*) FROM held_hooks WHERE hook_key = ${hook.hookKey})
< ${HOOK_INBOX_MAX_PER_HOOK}
AND (SELECT coalesce(sum(length(body)), 0) FROM held_hooks) + ${hook.body.byteLength}
<= ${HOOK_INBOX_MAX_BYTES}
ON CONFLICT (id) DO NOTHING
RETURNING id
`;
if (inserted.length === 0) {
yield* Effect.annotateCurrentSpan({ "relay.inbox.refused": yield* refusalReason(hook) });
return null;
}
yield* setTarget(baseUrl, { resetFailures: false });
const target = yield* readTarget;
const backlog = yield* backlogSize;
yield* Effect.annotateCurrentSpan({
"relay.inbox.held_count": backlog.count,
"relay.inbox.held_bytes": backlog.bytes,
});
return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(target?.failures ?? 0));
});
const backlogSize = Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
const rows = yield* sql<{ readonly count: number; readonly bytes: number }>`
SELECT count(*) AS count, coalesce(sum(length(body)), 0) AS bytes FROM held_hooks
`;
return rows[0] ?? { count: 0, bytes: 0 };
});
const refusalReason = Effect.fn("HookInboxStore.refusalReason")(function* (hook: HeldHook) {
const sql = yield* SqlClient.SqlClient;
const existing = yield* sql<{ readonly id: string }>`
SELECT id FROM held_hooks WHERE id = ${hook.id}
`;
if (existing.length > 0) return "already_held";
const backlog = yield* backlogSize;
if (backlog.count >= HOOK_INBOX_MAX_REQUESTS) return "max_requests";
if (backlog.bytes + hook.body.byteLength > HOOK_INBOX_MAX_BYTES) return "max_bytes";
return "max_per_hook";
});
export const wake = Effect.fn("HookInboxStore.wake")(function* (baseUrl: string) {
if (!(yield* hasPending)) return false;
yield* setTarget(baseUrl, { resetFailures: true });
return true;
});
export const clear = Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
yield* sql`DELETE FROM held_hooks`;
yield* sql`DELETE FROM held_hooks_target`;
});
export const deliverDue = Effect.fn("HookInboxStore.deliverDue")(function* <R>(
send: (baseUrl: string, hook: HeldHook) => Effect.Effect<DeliveryOutcome, never, R>,
) {
const sql = yield* SqlClient.SqlClient;
const startedAt = yield* Clock.currentTimeMillis;
const expired = yield* sql<{ readonly id: string }>`
DELETE FROM held_hooks WHERE received_at < ${iso(startedAt - HOOK_INBOX_TTL_MS)}
RETURNING id
`;
const target = yield* readTarget;
const readPage = (afterSeq: number, skip: ReadonlySet<string>) =>
sql<HeldHookRow>`
SELECT seq, id, received_at, method, raw_hook_id, raw_token, hook_key, query, headers, body
FROM held_hooks
WHERE seq > ${afterSeq}
${skip.size === 0 ? sql`` : sql`AND hook_key NOT IN ${sql.in([...skip])}`}
ORDER BY seq LIMIT ${ROWS_READ_PER_PAGE}
`;
let batch = yield* readPage(0, new Set());
if (batch.length === 0 || target === null) {
if (target === null) yield* sql`DELETE FROM held_hooks`;
else yield* resetFailures;
yield* Effect.annotateCurrentSpan({ "relay.inbox.expired": expired.length });
return null;
}
let failures = target.failures;
let sent = 0;
let delivered = 0;
let unreadable = 0;
let longestWaitMs = 0;
const busyHooks = new Set<string>();
// Delivery results and counts belong to the enclosing deliverDue span.
const annotateRun = Effect.fnUntraced(function* (result: string) {
const backlog = yield* backlogSize;
yield* Effect.annotateCurrentSpan({
"relay.inbox.run_result": result,
"relay.inbox.sent": sent,
"relay.inbox.delivered": delivered,
"relay.inbox.busy_hooks": busyHooks.size,
"relay.inbox.expired": expired.length,
"relay.inbox.unreadable": unreadable,
"relay.inbox.consecutive_failures": failures,
"relay.inbox.longest_wait_ms": longestWaitMs,
"relay.inbox.held_count": backlog.count,
"relay.inbox.held_bytes": backlog.bytes,
});
});
const stillHeld = (seq: number) =>
sql<{ readonly seq: number }>`SELECT seq FROM held_hooks WHERE seq = ${seq}`.pipe(
Effect.map((rows) => rows.length > 0),
);
// Exhaust non-busy requests before stopping so busy hooks cannot hide others' backlog.
pages: while (batch.length > 0) {
for (const row of batch) {
if (sent === DELIVERIES_PER_RUN) break pages;
if (busyHooks.has(row.hook_key)) continue;
// Clear can discard this prefetched page while an upstream send is in flight.
if (!(yield* stillHeld(row.seq))) {
yield* annotateRun("cleared");
return null;
}
const headers = decodeHeaders(row.headers);
if (Option.isNone(headers)) {
yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`;
unreadable += 1;
continue;
}
sent += 1;
const outcome = yield* send(target.base_url, {
id: row.id,
receivedAt: row.received_at,
method: row.method,
rawHookId: row.raw_hook_id,
rawToken: row.raw_token,
hookKey: row.hook_key,
query: row.query,
headers: headers.value,
body: row.body,
});
// A cleared request must neither advance this run nor schedule a retry.
if (!(yield* stillHeld(row.seq))) {
yield* annotateRun("cleared");
return null;
}
if (outcome === "unreachable") {
failures = yield* countFailure;
yield* annotateRun("unreachable");
return (yield* Clock.currentTimeMillis) + (yield* retryDelayMs(failures));
}
if (failures !== 0) {
failures = 0;
yield* resetFailures;
}
if (outcome === "busy") {
busyHooks.add(row.hook_key);
continue;
}
yield* sql`DELETE FROM held_hooks WHERE id = ${row.id}`;
delivered += 1;
const receivedAtMs = DateTime.toEpochMillis(DateTime.makeUnsafe(row.received_at));
longestWaitMs = Math.max(longestWaitMs, startedAt - receivedAtMs);
}
batch = yield* readPage(batch.at(-1)!.seq, busyHooks);
}
if (!(yield* hasPending)) {
yield* resetFailures;
yield* annotateRun("drained");
return null;
}
const now = yield* Clock.currentTimeMillis;
const idle = delivered === 0 && busyHooks.size > 0;
yield* annotateRun(idle ? "busy" : "more_pending");
return idle ? now + BUSY_RETRY_MS : now;
});