import * as aiGateway from "@distilled.cloud/cloudflare/ai-gateway";
import { deepEqual, isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import * as Effect from "effect/Effect";
import * as Predicate from "effect/Predicate";
import * as Stream from "effect/Stream";
import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
import type { Providers } from "../Providers.ts";
const TypeId = "Cloudflare.AI.DynamicRouting" as const;
type TypeId = typeof TypeId;
/**
* Reference to another element in the route graph.
*/
export type RouteEdge = {
/**
* The `id` of the element the edge points at.
*/
elementId: string;
};
/**
* Entry point of the route graph. Every route has exactly one.
*/
export type RouteStartElement = {
/**
* Unique element identifier within the route graph.
*/
id: string;
type: "start";
/**
* The element executed first.
*/
outputs: { next: RouteEdge };
};
/**
* Branches the request based on conditions over request metadata.
*/
export type RouteConditionalElement = {
/**
* Unique element identifier within the route graph.
*/
id: string;
type: "conditional";
/**
* Conditions evaluated against the request.
*/
properties: { conditions?: unknown };
/**
* Edges taken when the conditions evaluate true / false.
*/
outputs: { true: RouteEdge; false: RouteEdge };
};
/**
* Splits traffic across multiple edges by percentage.
*/
export type RoutePercentageElement = {
/**
* Unique element identifier within the route graph.
*/
id: string;
type: "percentage";
/**
* Percentage-weighted edges.
*/
outputs: Record<string, unknown>;
};
/**
* Applies a rate limit; requests over the limit take the fallback edge.
*/
export type RouteRateElement = {
/**
* Unique element identifier within the route graph.
*/
id: string;
type: "rate";
properties: {
/**
* Key the limit is bucketed by.
*/
key: string;
/**
* Maximum count/cost allowed inside the window.
*/
limit: number;
/**
* Whether the limit counts requests or cost.
*/
limitType: "count" | "cost";
/**
* Window size in seconds.
*/
window: number;
};
/**
* Edges taken when under (success) or over (fallback) the limit.
*/
outputs: { success: RouteEdge; fallback: RouteEdge };
};
/**
* Sends the request to a provider/model; failures take the fallback edge.
*/
export type RouteModelElement = {
/**
* Unique element identifier within the route graph.
*/
id: string;
type: "model";
properties: {
/**
* Provider slug (e.g. `workers-ai`, `openai`, `anthropic`).
*/
provider: string;
/**
* Model identifier (e.g. `@cf/meta/llama-3.1-8b-instruct`).
*/
model: string;
/**
* Number of retries before taking the fallback edge.
*/
retries: number;
/**
* Request timeout in milliseconds.
*/
timeout: number;
};
/**
* Edges taken on success / failure.
*/
outputs: { success: RouteEdge; fallback: RouteEdge };
};
/**
* Terminal element of the route graph.
*/
export type RouteEndElement = {
/**
* Unique element identifier within the route graph.
*/
id: string;
type: "end";
/**
* Always empty.
*/
outputs: Record<string, never>;
};
/**
* A node in an AI Gateway dynamic routing graph. A well-formed graph starts
* at a `start` element and every path terminates at an `end` element.
*/
export type RouteElement =
| RouteStartElement
| RouteConditionalElement
| RoutePercentageElement
| RouteRateElement
| RouteModelElement
| RouteEndElement;
export type GatewayDynamicRoutingProps = {
/**
* The AI Gateway the route belongs to. Changing the gateway triggers a
* replacement.
*/
gatewayId: string;
/**
* Route name, unique within the gateway. If omitted, a unique name is
* generated from the app, stage, and logical ID. Renames are applied in
* place.
* @default ${app}-${stage}-${id}
*/
name?: string;
/**
* The element graph describing how requests are routed. Changing the
* graph creates a new route version and deploys it.
*/
elements: RouteElement[];
};
export type GatewayDynamicRoutingAttributes = {
/**
* Server-generated route identifier. Stable across updates.
*/
routeId: string;
/**
* The Cloudflare account the route belongs to.
*/
accountId: string;
/**
* The AI Gateway the route belongs to.
*/
gatewayId: string;
/**
* Route name.
*/
name: string;
/**
* The element graph of the currently deployed route version.
*/
elements: RouteElement[];
/**
* Identifier of the currently deployed route version.
*/
versionId: string;
/**
* Identifier of the active deployment.
*/
deploymentId: string;
/**
* When the route was created.
*/
createdAt: string;
/**
* When the route was last modified.
*/
modifiedAt: string;
};
export type GatewayDynamicRouting = Resource<
TypeId,
GatewayDynamicRoutingProps,
GatewayDynamicRoutingAttributes,
never,
Providers
>;
/**
* A dynamic routing configuration ("route") on a Cloudflare.AI. Gateway.
*
* Dynamic routing models request handling as a graph of elements — start,
* conditional, percentage split, rate limit, model, and end nodes — so a
* single gateway endpoint can A/B test models, enforce per-user budgets, and
* fall back between providers without app changes.
*
* Cloudflare versions route configurations: changing `elements` creates a
* new version and deploys it; the reconciler also re-deploys when the live
* deployed version drifts from the desired graph. Renames are applied in
* place; only moving the route to a different gateway forces a replacement.
* ### Creating a Route
* **Example:** Route all traffic to one model
* ```typescript
* const gateway = yield* Cloudflare.AI.Gateway("Gateway");
*
* const route = yield* Cloudflare.AI.GatewayDynamicRouting("Llama", {
* gatewayId: gateway.gatewayId,
* elements: [
* { id: "start", type: "start", outputs: { next: { elementId: "model" } } },
* {
* id: "model",
* type: "model",
* properties: {
* provider: "workers-ai",
* model: "@cf/meta/llama-3.1-8b-instruct",
* retries: 1,
* timeout: 30000,
* },
* outputs: {
* success: { elementId: "end" },
* fallback: { elementId: "end" },
* },
* },
* { id: "end", type: "end", outputs: {} },
* ],
* });
* ```
*
* ### Updating a Route
* **Example:** Change the model — creates and deploys a new version
* ```typescript
* const route = yield* Cloudflare.AI.GatewayDynamicRouting("Llama", {
* gatewayId: gateway.gatewayId,
* elements: [
* { id: "start", type: "start", outputs: { next: { elementId: "model" } } },
* {
* id: "model",
* type: "model",
* properties: {
* provider: "workers-ai",
* model: "@cf/meta/llama-3.3-70b-instruct-fp8-fast",
* retries: 2,
* timeout: 60000,
* },
* outputs: {
* success: { elementId: "end" },
* fallback: { elementId: "end" },
* },
* },
* { id: "end", type: "end", outputs: {} },
* ],
* });
* ```
*
* @see https://developers.cloudflare.com/ai-gateway/features/dynamic-routing/
*
* @resource
* @product AI Gateway
* @category AI
*/
export const GatewayDynamicRouting = Resource<GatewayDynamicRouting>(TypeId, {
aliases: ["Cloudflare.AiGateway.DynamicRouting"],
});
/**
* Returns true if the given value is a GatewayDynamicRouting resource.
*/
export const isDynamicRouting = (value: unknown): value is GatewayDynamicRouting =>
Predicate.hasProperty(value, "Type") && value.Type === TypeId;
export const DynamicRoutingProvider = () =>
Provider.succeed(GatewayDynamicRouting, {
stables: ["routeId", "accountId", "gatewayId", "createdAt"],
diff: Effect.fn(function* ({ olds, news, output }) {
if (!isResolved(news)) return undefined;
const { accountId } = yield* yield* CloudflareEnvironment;
if ((output?.accountId ?? accountId) !== accountId) {
return { action: "replace" } as const;
}
// The gateway is a path parameter — a route cannot move between
// gateways in place. By diff time both sides are resolved strings.
const oldGatewayId = output?.gatewayId ?? olds?.gatewayId;
if (
typeof oldGatewayId === "string" &&
typeof news.gatewayId === "string" &&
oldGatewayId !== news.gatewayId
) {
return { action: "replace" } as const;
}
return undefined;
}),
read: Effect.fn(function* ({ id, olds, output }) {
const { accountId } = yield* yield* CloudflareEnvironment;
const acct = output?.accountId ?? accountId;
const gatewayId = output?.gatewayId ?? (olds?.gatewayId as string | undefined);
if (gatewayId === undefined) return undefined;
if (output?.routeId) {
const observed = yield* getRoute(acct, gatewayId, output.routeId);
return observed ? toAttributes(observed, acct) : undefined;
}
// Cold read — recover from lost state by matching the deterministic
// physical name. Route names are unique within a gateway, so an exact
// match identifies the route.
const name = yield* createRouteName(id, olds?.name);
const match = yield* findByName(acct, gatewayId, name);
if (match) {
const observed = yield* getRoute(acct, gatewayId, match.id);
return observed ? toAttributes(observed, acct) : undefined;
}
return undefined;
}),
reconcile: Effect.fn(function* ({ id, news, output }) {
const { accountId } = yield* yield* CloudflareEnvironment;
const gatewayId = news.gatewayId as string;
const name = yield* createRouteName(id, news.name);
const desired = news.elements ?? [];
// Observe — the routeId cached on `output` is a hint, not a
// guarantee: a missing route falls through and we recreate.
const observed = output?.routeId
? yield* getRoute(output.accountId ?? accountId, gatewayId, output.routeId)
: undefined;
// Ensure — create if missing. Route names are unique per gateway:
// tolerate the `RouteAlreadyExists` race (a peer reconciler created
// it concurrently, or state persistence failed after a previous
// create) by recovering the existing route by name.
const routeId =
observed?.id ??
(yield* aiGateway
.createDynamicRouting({
accountId,
gatewayId,
name,
elements: desired,
})
.pipe(
Effect.catchTag("RouteAlreadyExists", (error) =>
findByName(accountId, gatewayId, name).pipe(
Effect.flatMap((match) => (match ? Effect.succeed(match) : Effect.fail(error))),
),
),
Effect.map((route) => route.id),
));
// Sync — observed cloud state is the diff baseline. Creation above
// auto-deploys version 1, so a fresh `get` covers all paths.
const current = yield* aiGateway.getDynamicRouting({
accountId,
gatewayId,
id: routeId,
});
if (current.name !== name) {
yield* aiGateway
.patchDynamicRouting({
accountId,
gatewayId,
id: routeId,
name,
})
.pipe(
// Cloudflare does not cascade-delete a gateway's routes when the
// gateway itself is deleted: recreating a gateway with the same
// id resurrects its old route rows, and a rename onto one of
// those ghosts trips the backing `UNIQUE(routes.name,
// routes.gateway_id)` constraint (leaked as a 500, typed as
// `RouteAlreadyExists` via a distilled patch). Route names are
// unique and deterministic, so the holder is a stale duplicate
// of this logical resource — delete it and retry the rename.
Effect.catchTag("RouteAlreadyExists", (error) =>
Effect.gen(function* () {
const holder = yield* findByName(accountId, gatewayId, name);
if (holder === undefined || holder.id === routeId) {
return yield* error;
}
yield* aiGateway
.deleteDynamicRouting({
accountId,
gatewayId,
id: holder.id,
})
.pipe(Effect.catchTag("RouteNotFound", () => Effect.void));
yield* aiGateway.patchDynamicRouting({
accountId,
gatewayId,
id: routeId,
name,
});
}),
),
);
}
// `get` returns the *deployed* version's graph, so an undeployed
// draft version naturally shows up as drift and gets re-deployed.
const observedElements = elementsOf(current);
if (!deepEqual(observedElements, desired)) {
const version = yield* aiGateway.createVersionDynamicRouting({
accountId,
gatewayId,
id: routeId,
elements: desired,
});
yield* aiGateway.createDeploymentDynamicRouting({
accountId,
gatewayId,
id: routeId,
versionId: version.versionId,
});
}
// Return — re-read the final state so attributes reflect the
// deployed version, not stale intermediate responses.
const final = yield* aiGateway.getDynamicRouting({
accountId,
gatewayId,
id: routeId,
});
return toAttributes(final, accountId);
}),
delete: Effect.fn(function* ({ output }) {
yield* aiGateway
.deleteDynamicRouting({
accountId: output.accountId,
gatewayId: output.gatewayId,
id: output.routeId,
})
.pipe(
// Route already gone — or the whole parent gateway is gone.
Effect.catchTags({
RouteNotFound: () => Effect.void,
GatewayNotFound: () => Effect.void,
}),
);
}),
list: Effect.fn(function* () {
const { accountId } = yield* yield* CloudflareEnvironment;
// Routes are scoped under a gateway and there is no account-wide
// route list, so fan out: enumerate every account gateway, then
// exhaustively list each gateway's routes.
const gateways = yield* aiGateway.listAiGateways.pages({ accountId }).pipe(
Stream.runCollect,
Effect.map((chunk) => Array.from(chunk).flatMap((page) => page.result ?? [])),
);
const rows = yield* Effect.forEach(
gateways,
(gateway) =>
listRoutes(accountId, gateway.id).pipe(
// A gateway removed between enumeration and its route list is
// gone — skip it rather than failing the whole listing.
Effect.catchTag("GatewayNotFound", () =>
Effect.succeed([] as GatewayDynamicRoutingAttributes[]),
),
),
{ concurrency: 10 },
);
return rows.flat();
}),
});
/**
* Exhaustively list every route in a gateway. `listDynamicRoutings` is not a
* paginated distilled operation (the Cloudflare response carries only
* `page`/`per_page`, no total), so page manually until a short page signals
* the end.
*
* The list endpoint omits each route's element graph — its items carry only
* deployment/version metadata, no `version.data` — so hydrate every route with
* a per-route `get` to populate `elements` into the exact `read` shape.
*/
const listRoutes = (accountId: string, gatewayId: string) =>
Effect.gen(function* () {
const perPage = 50;
const ids: string[] = [];
for (let page = 1; ; page++) {
const response = yield* aiGateway.listDynamicRoutings({
accountId,
gatewayId,
page,
perPage,
});
const routes = response.data.routes;
for (const route of routes) {
ids.push(route.id);
}
if (routes.length < perPage) break;
}
const hydrated = yield* Effect.forEach(ids, (id) => getRoute(accountId, gatewayId, id), {
concurrency: 10,
});
return hydrated.filter((r) => r !== undefined).map((route) => toAttributes(route, accountId));
});
/**
* Read a route by id, mapping "gone" (`RouteNotFound`, Cloudflare error code
* 7005, or `GatewayNotFound`, code 7002, when the parent gateway was
* deleted) to `undefined`.
*/
const getRoute = (accountId: string, gatewayId: string, id: string) =>
aiGateway.getDynamicRouting({ accountId, gatewayId, id }).pipe(
Effect.catchTags({
RouteNotFound: () => Effect.undefined,
GatewayNotFound: () => Effect.undefined,
}),
);
/**
* Find a route by exact name. Route names are unique within a gateway. A
* missing parent gateway means the route is gone too.
*/
const findByName = (accountId: string, gatewayId: string, name: string) =>
aiGateway.listDynamicRoutings({ accountId, gatewayId, perPage: 50 }).pipe(
Effect.map((list) => list.data.routes.find((r) => r.name === name)),
Effect.catchTag("GatewayNotFound", () => Effect.undefined),
);
const createRouteName = (id: string, name: string | undefined) =>
Effect.gen(function* () {
return name ?? (yield* createPhysicalName({ id, lowercase: true }));
});
/**
* The deployed element graph. Cloudflare returns it as `version.data` (the
* distilled schema types it `unknown` because the public spec wrongly
* declares it a string).
*/
const elementsOf = (route: aiGateway.GetDynamicRoutingResponse): RouteElement[] =>
(route.version.data ?? []) as RouteElement[];
const toAttributes = (
route: aiGateway.GetDynamicRoutingResponse,
accountId: string,
): GatewayDynamicRoutingAttributes => ({
routeId: route.id,
accountId,
gatewayId: route.gatewayId,
name: route.name,
elements: elementsOf(route),
versionId: route.deployment.versionId,
deploymentId: route.deployment.deploymentId,
createdAt: route.createdAt,
modifiedAt: route.modifiedAt,
});