import * as Alchemy from "alchemy";
import * as Axiom from "@alchemy.run/axiom";
import * as Output from "alchemy/Output";
import * as Config from "effect/Config";
import * as Effect from "effect/Effect";
import * as Option from "effect/Option";
import { relayResourceNameForStage } from "./deploymentConfig.ts";
const RUNBOOK_URL =
"https://github.com/pingdotgg/t3code/blob/main/docs/operations/relay-observability.md#alerts";
const LINK_ROUTE = `"/v1/client/environment-links"`;
const TUNNEL_ROUTES = `"/v1/environments/:environmentId/tunnel", "/v1/environments/:environmentId/tunnel/recovery"`;
const CLOUDFLARE_API = `['attributes.url.full'] startswith "https://api.cloudflare.com"`;
const STATUS = `toint(['attributes.http.response.status_code'])`;
// The tunnel reaper records its counters on this span every five minutes,
// including while cleanup is off.
const SWEEP = `name == "relay.managed_endpoint_reaper.sweep"`;
const sweepCounter = (field: string) =>
`toint(['attributes.custom']['relay.managed_endpoint_reaper.${field}'])`;
// Status codes, not span errors: a request cut off by the relay deadline ends
// as an interrupted span with no error flag, but still answers 504.
// Below `minimum` requests the rate reads 0, so a quiet window cannot page.
const serverFailurePercent = (dataset: string, routes: string, minimum: number) =>
[
`['${dataset}']`,
`| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${routes})`,
`| summarize total = count(), failed = countif(${STATUS} >= 500)`,
`| project failure_pct = iff(total >= ${minimum}, 100.0 * failed / total, 0.0)`,
].join("\n");
type Severity = "page" | "warn";
interface RelayMonitor {
readonly id: string;
readonly severity: Severity;
readonly name: string;
/** Why it fires and the healthy baseline. The runbook link is appended. */
readonly description: string;
readonly query: (dataset: string) => string;
readonly operator: "Above" | "AboveOrEqual" | "Below";
readonly threshold: number;
readonly rangeMinutes: number;
readonly intervalMinutes?: number;
readonly triggerFromNRuns?: number;
readonly alertOnNoData?: boolean;
}
// Thresholds come from 2026-10-16..19 prod data: the worst healthy window,
// and the outage (Cloudflare API 429s stalling every provision).
// Paging monitors need two bad runs in a row; warnings catch the slow
// degradations an outage threshold has to ignore.
const relayMonitors: ReadonlyArray<RelayMonitor> = [
{
id: "RelayLinkFailureRateMonitor",
severity: "page",
name: "T3 Connect relay: links failing",
description:
"Share of POST /v1/client/environment-links (turning on T3 Connect) answering 5xx over 15 min. Healthy: median 11%, worst 75%. #16810: 87-96%.",
query: (dataset) => serverFailurePercent(dataset, LINK_ROUTE, 20),
operator: "Above",
threshold: 80,
rangeMinutes: 15,
triggerFromNRuns: 2,
},
{
id: "RelayTunnelFailureRateMonitor",
severity: "page",
name: "T3 Connect relay: tunnel recovery failing",
description:
"Share of host tunnel registration and recovery answering 5xx over 15 min. Hosts whose tunnel drops cannot come back. Healthy: median 4%, worst 52%. #16810: 83-94%.",
query: (dataset) => serverFailurePercent(dataset, TUNNEL_ROUTES, 200),
operator: "Above",
threshold: 60,
rangeMinutes: 15,
triggerFromNRuns: 2,
},
{
id: "RelayConnectReachabilityMonitor",
severity: "page",
name: "T3 Connect relay: environments unreachable",
description:
"Share of environments clients tried to reach that connected at least once in 30 min. Offline hosts keep this below 100%, and retry storms against them do not move it. Healthy: 65-72%.",
query: (dataset) =>
[
`['${dataset}']`,
`| where name == "relay.environment_connector.connect"`,
`| extend environment = tostring(['attributes.custom']['relay.environment_id'])`,
`| summarize environments = dcount(environment), connected = dcountif(environment, error != true)`,
`| project reachable_pct = iff(environments >= 1000, 100.0 * connected / environments, 100.0)`,
].join("\n"),
operator: "Below",
threshold: 55,
rangeMinutes: 30,
triggerFromNRuns: 2,
},
{
id: "RelayPushFailureRateMonitor",
severity: "page",
name: "T3 Connect relay: push delivery failing",
description:
"Share of APNs and FCM delivery jobs failing over 15 min, e.g. an expired APNs key or FCM credential. Healthy: APNs under 0.3%, FCM under 4%.",
query: (dataset) =>
[
`['${dataset}']`,
`| where name in ("relay.apns_deliveries.process_signed_job", "relay.fcm.process")`,
`| summarize total = count(), failed = countif(error == true)`,
`| project failure_pct = iff(total >= 100, 100.0 * failed / total, 0.0)`,
].join("\n"),
operator: "Above",
threshold: 10,
rangeMinutes: 15,
triggerFromNRuns: 2,
},
{
id: "RelayCloudflareRateLimitedMonitor",
severity: "page",
name: "T3 Connect relay: Cloudflare API rate limited",
description:
"Cloudflare API 429s, plus calls the relay refused while paused after one, per 10 min. Sustained at this level, links and recoveries stall. Peak 429s before #16810: 1,286; during it: ~11k.",
query: (dataset) =>
`['${dataset}']\n| where (${CLOUDFLARE_API} and ${STATUS} == 429) or name == "relay.cloudflare_api_pause.rejected"\n| summarize rate_limited = count()`,
operator: "AboveOrEqual",
threshold: 500,
rangeMinutes: 10,
triggerFromNRuns: 2,
},
{
id: "RelayTrafficDroppedMonitor",
severity: "page",
name: "T3 Connect relay: traffic dropped",
description:
"Relay requests per 10 min collapsed: Worker down, broken deploy, or trace export stopped. Healthy minimum: ~69k.",
query: (dataset) => `['${dataset}']\n| where kind == "server"\n| summarize requests = count()`,
operator: "Below",
threshold: 10_000,
rangeMinutes: 10,
triggerFromNRuns: 2,
alertOnNoData: true,
},
{
id: "RelayCloudflareRateLimitWarningMonitor",
severity: "warn",
name: "T3 Connect relay: Cloudflare API near its limit",
description:
"Any sustained Cloudflare API 429s or paused calls. 429 bursts ran for days before #16810; raise the account limit or cut call volume before they grow.",
query: (dataset) =>
`['${dataset}']\n| where (${CLOUDFLARE_API} and ${STATUS} == 429) or name == "relay.cloudflare_api_pause.rejected"\n| summarize rate_limited = count()`,
operator: "AboveOrEqual",
threshold: 20,
rangeMinutes: 10,
triggerFromNRuns: 2,
},
{
id: "RelayLinkChronicFailureMonitor",
severity: "warn",
name: "T3 Connect relay: links degraded",
description:
"Share of links answering 5xx over 6 h, mostly 9 s deadline 504s from slow provisioning. Healthy days: 10-18%. Degraded days before #16810: 28-33%.",
query: (dataset) => serverFailurePercent(dataset, LINK_ROUTE, 100),
operator: "Above",
threshold: 20,
rangeMinutes: 360,
intervalMinutes: 30,
},
{
id: "RelayDatabaseLatencyMonitor",
severity: "warn",
name: "T3 Connect relay: database slow",
description:
"p90 Postgres query time over 15 min, in ms. Every relay request runs several queries in sequence, so this sets how close provisioning gets to the 9 s deadline. Healthy: ~200 ms.",
query: (dataset) =>
`['${dataset}']\n| where name == "postgres"\n| summarize p90 = percentile(duration, 90)\n| project p90_ms = p90 / 1000000`,
operator: "Above",
threshold: 500,
rangeMinutes: 15,
triggerFromNRuns: 2,
},
{
id: "RelayTunnelCleanupStoppedMonitor",
severity: "warn",
name: "T3 Connect relay: tunnel cleanup stopped",
description:
"Tunnel cleanup sweeps that finished in 30 min. Healthy: 6. The cron is not running, or every sweep fails, as during the 2026-10-19 Cloudflare 429s when listing tunnels timed out. Idle tunnels pile up and are billed meanwhile.",
query: (dataset) =>
`['${dataset}']\n| where ${SWEEP} and error != true\n| summarize sweeps = count()`,
operator: "Below",
threshold: 1,
rangeMinutes: 30,
triggerFromNRuns: 2,
alertOnNoData: true,
},
{
id: "RelayTunnelCleanupFailingMonitor",
severity: "warn",
name: "T3 Connect relay: tunnel deletions failing",
description:
"Cloudflare tunnel deletions that failed in 30 min. Healthy: 0; the worst 30 min so far had 3. The relay logs a warning with the Cloudflare error for each.",
query: (dataset) =>
`['${dataset}']\n| where ${SWEEP}\n| summarize failed = sum(${sweepCounter("failed")})`,
operator: "Above",
threshold: 10,
rangeMinutes: 30,
triggerFromNRuns: 2,
},
];
interface QueryChart {
readonly id: string;
readonly name: string;
readonly type: "TimeSeries" | "Table";
readonly query: { readonly apl: string };
}
const timeSeries = (id: string, name: string, apl: string): QueryChart => ({
id,
name,
type: "TimeSeries",
query: { apl },
});
const relayDashboardCharts = (dataset: string): ReadonlyArray<QueryChart> => {
const source = `['${dataset}']`;
const linkRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] == ${LINK_ROUTE}`;
const tunnelRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${TUNNEL_ROUTES})`;
const clientConnections = `${source}\n| where name == "relay.connection.attempt" and error != true\n| extend surface = tostring(['resource.custom']['t3.client.surface'])\n| where isnotempty(surface)`;
return [
timeSeries(
"links",
"Links by outcome",
`${linkRequests}\n| summarize succeeded = countif(${STATUS} < 400), rejected = countif(${STATUS} >= 400 and ${STATUS} < 500), failed = countif(${STATUS} >= 500) by bin_auto(_time)`,
),
timeSeries(
"link-latency",
"Link latency (9 s deadline)",
`${linkRequests}\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time)`,
),
timeSeries(
"reachability",
"Environments reachable per 30 min (%)",
`${source}\n| where name == "relay.environment_connector.connect"\n| extend environment = tostring(['attributes.custom']['relay.environment_id'])\n| summarize environments = dcount(environment), connected = dcountif(environment, error != true) by bin(_time, 30m)\n| project _time, reachable_pct = 100.0 * connected / environments`,
),
timeSeries(
"tunnel",
"Tunnel registration and recovery by outcome",
`${tunnelRequests}\n| summarize succeeded = countif(${STATUS} < 400), rejected = countif(${STATUS} >= 400 and ${STATUS} < 500), failed = countif(${STATUS} >= 500) by bin_auto(_time)`,
),
timeSeries(
"cloudflare-api",
"Cloudflare API calls by status",
// Calls refused during a rate-limit pause never reach Cloudflare, so
// they are their own series or a storm would look like a quiet period.
`${source}\n| where ${CLOUDFLARE_API} or name == "relay.cloudflare_api_pause.rejected"\n| summarize count() by bin_auto(_time), status = case(name == "relay.cloudflare_api_pause.rejected", "paused", isnull(${STATUS}), "no response", tostring(${STATUS}))`,
),
timeSeries(
"provision-paths",
"Tunnel provisions by path",
`${source}\n| where name == "relay.managed_endpoint_provider.provision"\n| summarize reused_ready_tunnel = countif(error != true and tobool(['attributes.custom']['relay.managed_endpoint.reused_ready_tunnel']) == true), full_provision = countif(error != true and isnull(['attributes.custom']['relay.managed_endpoint.reused_ready_tunnel'])), failed = countif(error == true) by bin_auto(_time)`,
),
timeSeries(
"postgres",
"Postgres query time",
`${source}\n| where name == "postgres"\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time)`,
),
timeSeries(
"push",
"Push delivery jobs",
`${source}\n| where name in ("relay.apns_deliveries.process_signed_job", "relay.fcm.process")\n| summarize delivered = countif(error != true), failed = countif(error == true) by bin_auto(_time), service = iff(name == "relay.fcm.process", "fcm", "apns")`,
),
timeSeries(
"tunnel-cleanup",
"Tunnel cleanup per sweep",
`${source}\n| where ${SWEEP}\n| summarize deleted = sum(${sweepCounter("deleted")}), deleted_legacy = sum(${sweepCounter("deletedLegacy")}), would_delete = sum(${sweepCounter("wouldDelete")}), would_delete_legacy = sum(${sweepCounter("wouldDeleteLegacy")}), failed = sum(${sweepCounter("failed")}), sweeps_failed = countif(error == true) by bin_auto(_time)`,
),
// Cloudflare's own counts for this stage, including tunnels the reaper
// skips, so this is the idle tunnels still billed.
timeSeries(
"tunnel-backlog",
"Idle tunnels in Cloudflare",
`${source}\n| where ${SWEEP}\n| summarize down = max(${sweepCounter("totalDown")}), never_connected = max(${sweepCounter("totalInactive")}) by bin_auto(_time)`,
),
timeSeries(
"server-errors",
"5xx by route",
`${source}\n| where kind == "server" and ${STATUS} >= 500\n| summarize count() by bin_auto(_time), route = tostring(['attributes.http.route']), status = ${STATUS}`,
),
{
id: "link-failures",
name: "Link failures by reason",
type: "Table",
query: {
apl: `${source}\n| where name == "relay.environment_linker.link" and error == true\n| extend message = tostring(['status.message'])\n| summarize failures = count() by reason = case(message contains "failed during '", strcat("provisioning: ", extract("failed during '([^']+)'", 1, message)), message startswith "Managed tunnel limit reached", "managed tunnel limit reached", substring(message, 0, 80))\n| top 10 by failures`,
},
},
// Connecting, as users see it: web, desktop and mobile export their
// connection spans into this dataset, in the same traces as the relay's
// and the host's spans. An attempt lasts until initial sync finishes.
timeSeries(
"connect-time",
"Connecting: time to connected by surface",
`${clientConnections}\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time), surface`,
),
timeSeries(
"connect-outcomes",
"Connecting: attempts by outcome",
`${source}\n| where name == "relay.connection.attempt" and isnotempty(['resource.custom']['t3.client.surface'])\n| summarize connected = countif(error != true), failed = countif(error == true) by bin_auto(_time)`,
),
{
id: "connect-phases",
name: "Connecting: where the time goes",
type: "Table",
query: {
// Failed direct route checks stay in: waiting out an unreachable LAN
// address before falling back to the relay is the slowest step.
apl: [
source,
`| where isnotempty(['resource.custom']['t3.client.surface'])`,
`| extend url = tostring(['attributes.url.full'])`,
`| extend phase = case(`,
` name == "ConnectionDriver.checkRoute", "0. Direct route check, before any relay fallback",`,
` name == "relay.connection.cloudSessionToken.resolve", "1. Clerk session token",`,
` name == "clientRuntime.managedRelay.exchangeAccessToken", "2. Relay token exchange",`,
` kind == "client" and url startswith "https://relay.t3.codes/v1/environments/" and url endswith "/connect", "3. Relay connect, minting a host credential",`,
` name == "environment.authorization.descriptor", "4. Host descriptor",`,
` name == "environment.authorization.accessToken.exchange", "5. Host token exchange",`,
` name == "clientRuntime.connection.remote.createDpopSocketUrl", "6. Socket ticket",`,
` name == "environment.initialSync", "7. Initial sync",`,
` name == "relay.connection.attempt", "Total: time to connected",`,
` "")`,
`| where phase != "" and (error != true or phase startswith "0.")`,
`| summarize calls = count(), p50 = percentile(duration, 50), p90 = percentile(duration, 90) by phase`,
`| order by phase asc`,
].join("\n"),
},
},
timeSeries(
"relay-latency-by-continent",
"Relay token and connect latency by user continent (p50)",
`${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and (['attributes.http.route'] == "/v1/client/dpop-token" or ['attributes.http.route'] == "/v1/environments/:environmentId/connect")\n| extend continent = tostring(['attributes.custom']['relay.edge.continent'])\n| where isnotempty(continent)\n| summarize p50 = percentile(duration, 50) by bin_auto(_time), continent`,
),
];
};
export const notifierConfig = (name: string) =>
Config.String(name).pipe(
Config.option,
Config.map(Option.flatMap((value) => Option.liftPredicate(value.trim(), Boolean))),
);
/**
* A deployment opts in by naming the notifier that pages (incident.io for
* prod). Personal stages and forks leave it unset and get no monitors or
* dashboard. Warnings go to their own notifier when one is set, and are only
* visible in Axiom otherwise.
*/
export const RelayMonitoring = Effect.fn("RelayMonitoring")(function* (traces: Axiom.Dataset) {
const { stage } = yield* Alchemy.Stack;
const page = yield* notifierConfig("RELAY_ALERT_NOTIFIER_ID");
if (Option.isNone(page)) return;
const notifiers: Record<Severity, Option.Option<string>> = {
page,
warn: yield* notifierConfig("RELAY_WARNING_NOTIFIER_ID"),
};
for (const monitor of relayMonitors) {
yield* Axiom.Monitor(monitor.id, {
name: stage === "prod" ? monitor.name : `[${stage}] ${monitor.name}`,
description: `${monitor.description} Runbook: ${RUNBOOK_URL}`,
type: "Threshold",
aplQuery: Output.map(traces.name, monitor.query),
operator: monitor.operator,
threshold: monitor.threshold,
intervalMinutes: monitor.intervalMinutes ?? 5,
rangeMinutes: monitor.rangeMinutes,
triggerFromNRuns: monitor.triggerFromNRuns ?? 1,
alertOnNoData: monitor.alertOnNoData ?? false,
notifierIds: Option.toArray(notifiers[monitor.severity]),
});
}
yield* Axiom.Dashboard("RelayHealthDashboard", {
dashboard: Output.map(traces.name, (dataset) => {
const charts = relayDashboardCharts(dataset);
return {
name: relayResourceNameForStage("t3-code-relay-health", stage),
// Shared with the whole org. An empty owner only means that for API
// tokens; deploys with a personal token would make it private.
owner: "X-AXIOM-EVERYONE",
description: `T3 Connect relay health. Alerts and runbook: ${RUNBOOK_URL}`,
refreshTime: 60,
schemaVersion: 2,
timeWindowStart: "qr-now-24h",
timeWindowEnd: "qr-now",
charts: [...charts],
// Two charts per row on Axiom's 12-column grid.
layout: charts.map((chart, index) => ({
i: chart.id,
x: (index % 2) * 6,
y: Math.floor(index / 2) * 6,
w: 6,
h: 6,
})),
};
}),
});
});