/// <reference types="@cloudflare/workers-types" />
import type { RuntimeContext } from "alchemy/RuntimeContext";
import {
AiError,
LanguageModel as AiLanguageModel,
IdGenerator,
Prompt,
Response,
Tool,
} from "effect/ai";
import * as Effect from "effect/Effect";
import * as Sse from "effect/encoding/Sse";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";
/**
* The slice of an AI client the LanguageModel adapter needs: the raw Workers
* AI handle plus, when routed through an AI Gateway, the gateway id.
*
* Both `Cloudflare.AI.QueryGateway(gateway)` (gateway-routed) and
* `Cloudflare.Workers.AI()` (plain Workers AI binding) clients satisfy it.
*/
export interface LanguageModelClient {
/** Effect resolving to the raw Workers AI runtime binding. */
readonly raw: Effect.Effect<Ai, never, RuntimeContext>;
/**
* Effect resolving to the AI Gateway id to route `ai.run` calls through.
* Omit for a plain Workers AI binding — requests then go directly to
* Workers AI without a gateway.
*/
readonly id?: Effect.Effect<string, never, RuntimeContext>;
}
/**
* Options for constructing a Workers AI-backed LanguageModel.
*/
export interface LanguageModelOptions {
/**
* Already-bound AI client — from `Cloudflare.AI.QueryGateway(gateway)`
* (routed through the gateway) or `Cloudflare.Workers.AI()` (direct).
*/
readonly client: LanguageModelClient;
/** Workers AI model id, e.g. `@cf/meta/llama-3.3-70b-instruct-fp8-fast`. */
readonly model: string;
/** Optional per-call defaults; overridable per request via `providerOptions`. */
readonly parameters?: {
readonly temperature?: number;
readonly maxTokens?: number;
readonly topP?: number;
readonly topK?: number;
readonly seed?: number;
readonly frequencyPenalty?: number;
readonly presencePenalty?: number;
};
}
/**
* Provide a {@link AiLanguageModel.LanguageModel} layer backed by the supplied
* AI Gateway client and Workers AI model.
*/
export const makeLanguageModelLayer = (
options: LanguageModelOptions,
): Layer.Layer<AiLanguageModel.LanguageModel, never, RuntimeContext> =>
Layer.effect(AiLanguageModel.LanguageModel, makeLanguageModel(options));
/**
* Build a {@link AiLanguageModel.LanguageModel} that proxies generateText/streamText
* through the supplied AI Gateway client to a Workers AI model.
*/
export const makeLanguageModel = ({
client,
model,
parameters,
}: LanguageModelOptions): Effect.Effect<AiLanguageModel.LanguageModel, never, RuntimeContext> =>
Effect.gen(function* () {
const ai = yield* client.raw;
const gatewayId = client.id === undefined ? undefined : yield* client.id;
const startsInThink = THINK_PREFILLED_MODELS.has(model);
const callRaw = (
body: WorkersAiInputs,
method: "generateText" | "streamText",
): Effect.Effect<Response, AiError.AiError> =>
Effect.tryPromise({
try: () =>
ai.run(model as keyof AiModels, body as unknown as AiModels[keyof AiModels]["inputs"], {
...(gatewayId === undefined ? {} : { gateway: { id: gatewayId } }),
returnRawResponse: true,
}),
catch: (cause) => toAiError(cause, method),
});
return yield* AiLanguageModel.make({
generateText: (options) =>
Effect.gen(function* () {
const body = toRequestBody({ options, parameters, stream: false });
const resp = yield* callRaw(body, "generateText");
const json = yield* Effect.tryPromise({
try: () => resp.json() as Promise<Record<string, unknown>>,
catch: (cause) => toAiError(cause, "generateText"),
});
return yield* parseGenerateText(json, startsInThink);
}),
streamText: (options) =>
Stream.unwrap(
Effect.gen(function* () {
const idGen = yield* IdGenerator.IdGenerator;
const body = toRequestBody({ options, parameters, stream: true });
const resp = yield* callRaw(body, "streamText");
const hasTools = options.tools.length > 0 && options.toolChoice !== "none";
return parseStreamText(resp, idGen, hasTools, startsInThink);
}),
),
});
});
// ---------------------------------------------------------------------------
// Wire format types (Workers AI request)
//
// Workers AI returns two response shapes depending on the model:
// - Native: { response: "...", tool_calls: [...] , usage }
// - OpenAI: { choices: [{ message: { content, tool_calls, reasoning_content } }], usage }
//
// We accept both defensively — schemas would over-constrain.
// ---------------------------------------------------------------------------
interface WorkersAiMessage {
readonly role: "system" | "user" | "assistant" | "tool";
readonly content?: unknown;
readonly name?: string;
readonly tool_call_id?: string;
readonly tool_calls?: ReadonlyArray<{
readonly id: string;
readonly type: "function";
readonly function: { readonly name: string; readonly arguments: string };
}>;
readonly reasoning?: string;
}
interface WorkersAiToolDef {
readonly type: "function";
readonly function: {
readonly name: string;
readonly description?: string;
readonly parameters: unknown;
};
}
type WorkersAiToolChoice =
| "auto"
| "required"
| "none"
| { readonly type: "function"; readonly function: { readonly name: string } };
interface WorkersAiInputs {
readonly messages: ReadonlyArray<WorkersAiMessage>;
readonly tools?: ReadonlyArray<WorkersAiToolDef>;
readonly tool_choice?: WorkersAiToolChoice;
readonly stream?: boolean;
readonly stream_options?: { readonly include_usage: boolean };
readonly max_tokens?: number;
readonly temperature?: number;
readonly top_p?: number;
readonly top_k?: number;
readonly random_seed?: number;
readonly frequency_penalty?: number;
readonly presence_penalty?: number;
}
// ---------------------------------------------------------------------------
// Prompt → Workers AI messages (pure, no .push mutation)
// ---------------------------------------------------------------------------
const uint8ArrayToBase64 = (bytes: Uint8Array): string => {
// Keep older Workers and no_nodejs_compat deployments working without a
// Node import. The native web API encodes only this view's byte range.
const toBase64 = (bytes as Uint8Array & { toBase64?: () => string }).toBase64;
if (typeof toBase64 === "function") return toBase64.call(bytes);
let binary = "";
const chunkSize = 8192;
for (let i = 0; i < bytes.length; i += chunkSize) {
const chunk = bytes.subarray(i, Math.min(i + chunkSize, bytes.length));
binary += String.fromCharCode(...chunk);
}
return btoa(binary);
};
const fileToImageUrl = (data: string | Uint8Array | URL, mediaType: string): string => {
if (data instanceof URL) return data.toString();
if (data instanceof Uint8Array) {
return `data:${mediaType};base64,${uint8ArrayToBase64(data)}`;
}
if (data.startsWith("data:") || data.startsWith("http")) return data;
return `data:${mediaType};base64,${data}`;
};
const convertPromptToMessages = (prompt: Prompt.Prompt): ReadonlyArray<WorkersAiMessage> =>
prompt.content.flatMap((m): ReadonlyArray<WorkersAiMessage> => {
switch (m.role) {
case "system":
return [{ role: "system", content: m.content }];
case "user":
return [toUserMessage(m.content)];
case "assistant":
return [toAssistantMessage(m.content)];
case "tool":
return m.content.flatMap(toToolMessage);
}
});
const toUserMessage = (parts: Prompt.UserMessage["content"]): WorkersAiMessage => {
const text = parts.flatMap((p) => (p.type === "text" ? [p.text] : [])).join("\n");
const images = parts.flatMap((p) =>
p.type === "file"
? [
{
type: "image_url" as const,
image_url: { url: fileToImageUrl(p.data, p.mediaType) },
},
]
: [],
);
if (images.length === 0) return { role: "user", content: text };
return {
role: "user",
content: [...(text.length > 0 ? [{ type: "text" as const, text }] : []), ...images],
};
};
const toAssistantMessage = (parts: Prompt.AssistantMessage["content"]): WorkersAiMessage => {
const text = parts.flatMap((p) => (p.type === "text" ? [p.text] : [])).join("");
const reasoning = parts.flatMap((p) => (p.type === "reasoning" ? [p.text] : [])).join("");
const toolCalls = parts.flatMap((p) =>
p.type === "tool-call"
? [
{
id: p.id,
type: "function" as const,
function: {
name: p.name,
arguments: typeof p.params === "string" ? p.params : JSON.stringify(p.params ?? {}),
},
},
]
: [],
);
return {
role: "assistant",
content: text,
...(reasoning ? { reasoning } : {}),
...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),
};
};
const toToolMessage = (
part: Prompt.ToolMessage["content"][number],
): ReadonlyArray<WorkersAiMessage> =>
part.type === "tool-result"
? [
{
role: "tool",
name: part.name,
tool_call_id: part.id,
content: typeof part.result === "string" ? part.result : JSON.stringify(part.result),
},
]
: [];
// ---------------------------------------------------------------------------
// Tools / tool_choice
// ---------------------------------------------------------------------------
type PreparedToolSchema = {
readonly schema: Tool.Any["parametersSchema"];
readonly ast: Tool.Any["parametersSchema"]["ast"];
readonly json: ReturnType<typeof Tool.getJsonSchema>;
};
// Tools and Effect schemas expose immutable update methods. Still check both
// identities: JavaScript callers can replace a tool's schema in place. Dynamic
// JSON schemas remain caller-owned and may change without an identity change.
const preparedToolSchemas = new WeakMap<Tool.Any, PreparedToolSchema>();
const prepareTool = (tool: Tool.Any): WorkersAiToolDef => {
let parameters: ReturnType<typeof Tool.getJsonSchema>;
if (Tool.isDynamic(tool) && tool.jsonSchema !== undefined) {
parameters = Tool.getJsonSchema(tool);
} else {
const schema = tool.parametersSchema;
let cached = preparedToolSchemas.get(tool);
if (cached === undefined || cached.schema !== schema || cached.ast !== schema.ast) {
cached = { schema, ast: schema.ast, json: Tool.getJsonSchema(tool) };
preparedToolSchemas.set(tool, cached);
}
// Each request previously owned its compiled schema. Keep that isolation
// if the runtime client mutates request input, without recompiling it.
parameters = structuredClone(cached.json);
}
return {
type: "function",
function: {
name: tool.name,
description: Tool.getDescription(tool),
parameters,
},
};
};
const prepareTools = (
tools: ReadonlyArray<Tool.Any>,
toolChoice: AiLanguageModel.ProviderOptions["toolChoice"],
): {
tools?: ReadonlyArray<WorkersAiToolDef>;
tool_choice?: WorkersAiToolChoice;
} => {
if (tools.length === 0) return {};
if (toolChoice != null && typeof toolChoice === "object" && "tool" in toolChoice) {
return {
tools: tools.filter((tool) => tool.name === toolChoice.tool).map(prepareTool),
tool_choice: "required",
};
}
if (toolChoice != null && typeof toolChoice === "object" && "oneOf" in toolChoice) {
const allowed = new Set(toolChoice.oneOf);
return {
tools: tools.filter((tool) => allowed.has(tool.name)).map(prepareTool),
tool_choice: toolChoice.mode === "required" ? "required" : "auto",
};
}
return {
tools: tools.map(prepareTool),
tool_choice: toolChoice === "none" || toolChoice === "required" ? toolChoice : "auto",
};
};
// ---------------------------------------------------------------------------
// Request body
// ---------------------------------------------------------------------------
const toRequestBody = ({
options,
parameters,
stream,
}: {
readonly options: AiLanguageModel.ProviderOptions;
readonly parameters: LanguageModelOptions["parameters"];
readonly stream: boolean;
}): WorkersAiInputs => {
const messages = convertPromptToMessages(options.prompt);
const { tools, tool_choice } = prepareTools(options.tools, options.toolChoice);
return {
messages,
...(tools !== undefined ? { tools } : {}),
...(tool_choice !== undefined ? { tool_choice } : {}),
// `stream_options.include_usage` is the OpenAI-compatible opt-in for
// usage tokens to appear in the final streamed chunk. Without it most
// Workers AI models omit `usage` from the stream entirely, leaving the
// `finish` part with zeroed counts.
...(stream ? { stream: true, stream_options: { include_usage: true } } : {}),
...(parameters?.maxTokens !== undefined ? { max_tokens: parameters.maxTokens } : {}),
...(parameters?.temperature !== undefined ? { temperature: parameters.temperature } : {}),
...(parameters?.topP !== undefined ? { top_p: parameters.topP } : {}),
...(parameters?.topK !== undefined ? { top_k: parameters.topK } : {}),
...(parameters?.seed !== undefined ? { random_seed: parameters.seed } : {}),
...(parameters?.frequencyPenalty !== undefined
? { frequency_penalty: parameters.frequencyPenalty }
: {}),
...(parameters?.presencePenalty !== undefined
? { presence_penalty: parameters.presencePenalty }
: {}),
};
};
// ---------------------------------------------------------------------------
// Finish reason / usage mapping
// ---------------------------------------------------------------------------
const mapFinishReason = (raw: unknown): Response.FinishReason => {
switch (raw) {
case "stop":
return "stop";
case "length":
case "model_length":
return "length";
case "tool_calls":
return "tool-calls";
case "content_filter":
case "content-filter":
return "content-filter";
case "error":
return "error";
case undefined:
case null:
return "unknown";
default:
return "other";
}
};
const finishMetadata = (
raw: Record<string, unknown> | undefined,
): { readonly metadata?: Response.FinishPartMetadata } => {
const neurons = (raw?.usage as Record<string, unknown> | undefined)?.neurons;
return typeof neurons === "number" && Number.isFinite(neurons)
? { metadata: { cloudflare: { neurons } } }
: {};
};
/**
* The neurons Workers AI billed for a response, read from its finish part.
* Workers AI reports this as `usage.neurons`; the adapter carries it on the
* finish part as `metadata.cloudflare.neurons`. `undefined` when the model
* did not report it.
*
* @example
* ```typescript
* const response = yield* LanguageModel.generateText({ prompt });
* const finish = response.content.find((part) => part.type === "finish");
* const neurons = finish ? Cloudflare.AI.finishNeurons(finish) : undefined;
* ```
*/
export const finishNeurons = (part: {
readonly metadata: Response.FinishPartMetadata;
}): number | undefined =>
Option.getOrUndefined(Option.map(decodeNeurons(part.metadata), (m) => m.cloudflare.neurons));
const decodeNeurons = Schema.decodeUnknownOption(
Schema.Struct({ cloudflare: Schema.Struct({ neurons: Schema.Number }) }),
);
const mapUsage = (raw: Record<string, unknown> | undefined): Response.Usage => {
const usage = (raw?.usage as Record<string, unknown> | undefined) ?? {};
const promptTokens = (usage.prompt_tokens as number | undefined) ?? 0;
const completionTokens = (usage.completion_tokens as number | undefined) ?? 0;
const cached = (usage.prompt_tokens_details as { cached_tokens?: number } | undefined)
?.cached_tokens;
// Construct an actual `Response.Usage` instance — `Schema.Class<Usage>`
// encodes by going through the class constructor / `isInstance` check, so a
// plain struct that "matches" the encoded shape isn't enough.
return Response.Usage.make({
inputTokens: {
uncached: cached !== undefined ? Math.max(0, promptTokens - cached) : promptTokens,
total: promptTokens,
cacheRead: cached ?? 0,
cacheWrite: 0,
},
outputTokens: {
total: completionTokens,
text: 0,
reasoning: 0,
},
});
};
// ---------------------------------------------------------------------------
// Inline reasoning
//
// Some models (e.g. `@cf/deepseek-ai/deepseek-r1-distill-qwen-32b`) write
// their chain of thought into the text itself, as `<think>…</think>`
// followed by the answer. Others (e.g. `@cf/qwen/qwq-32b`) have a chat
// template that pre-opens `<think>`, so only the closing tag is generated.
// `splitThink` incrementally routes text before `</think>` to reasoning and
// the rest to text, holding back any suffix that could be a split tag.
// ---------------------------------------------------------------------------
/**
* Models whose chat template opens the `<think>` block in the prompt, so
* their output starts inside reasoning and only `</think>` is generated.
*/
const THINK_PREFILLED_MODELS: ReadonlySet<string> = new Set(["@cf/qwen/qwq-32b"]);
const THINK_OPEN = "<think>";
const THINK_CLOSE = "</think>";
interface ThinkState {
/**
* - `pending`: no non-whitespace text yet; a leading `<think>` opens reasoning
* - `inside`: inside the `<think>` block
* - `after`: just closed the block; drop the whitespace separating the answer
* - `outside`: plain text
*/
readonly mode: "pending" | "inside" | "after" | "outside";
/** Held-back text that may be the start of a tag split across chunks. */
readonly carry: string;
}
const initialThinkState = (startsInThink: boolean): ThinkState => ({
mode: startsInThink ? "inside" : "pending",
carry: "",
});
/** Length of the longest suffix of `s` that is a proper prefix of `tag`. */
const partialTagSuffix = (s: string, tag: string): number => {
for (let n = Math.min(tag.length - 1, s.length); n > 0; n--) {
if (s.endsWith(tag.slice(0, n))) return n;
}
return 0;
};
const splitThink = (
state: ThinkState,
input: string,
): { readonly state: ThinkState; readonly reasoning: string; readonly text: string } => {
let buf = state.carry + input;
let mode = state.mode;
let reasoning = "";
while (true) {
switch (mode) {
case "pending": {
const trimmed = buf.trimStart();
if (trimmed.startsWith(THINK_OPEN)) {
buf = trimmed.slice(THINK_OPEN.length);
mode = "inside";
continue;
}
if (THINK_OPEN.startsWith(trimmed)) {
return { state: { mode, carry: buf }, reasoning, text: "" };
}
mode = "outside";
continue;
}
case "inside": {
const end = buf.indexOf(THINK_CLOSE);
if (end >= 0) {
reasoning += buf.slice(0, end);
buf = buf.slice(end + THINK_CLOSE.length);
mode = "after";
continue;
}
const held = partialTagSuffix(buf, THINK_CLOSE);
reasoning += buf.slice(0, buf.length - held);
return { state: { mode, carry: buf.slice(buf.length - held) }, reasoning, text: "" };
}
case "after": {
buf = buf.trimStart();
if (buf.length === 0) return { state: { mode, carry: "" }, reasoning, text: "" };
mode = "outside";
continue;
}
case "outside":
return { state: { mode, carry: "" }, reasoning, text: buf };
}
}
};
/**
* Split a complete response text. A `</think>` with no opening tag means the
* template pre-opened the block, so everything before it is reasoning.
*/
const splitThinkText = (
text: string,
startsInThink: boolean,
): { readonly reasoning: string; readonly text: string } => {
const prefilled =
startsInThink || (text.includes(THINK_CLOSE) && !text.trimStart().startsWith(THINK_OPEN));
const split = splitThink(initialThinkState(prefilled), text);
return split.state.mode === "inside"
? { reasoning: split.reasoning + split.state.carry, text: split.text }
: { reasoning: split.reasoning, text: split.text + split.state.carry };
};
// ---------------------------------------------------------------------------
// generateText: JSON → Response.PartEncoded[]
//
// Normalize the dual-shape (native + OpenAI) response into a single
// `DecodedResponse` once, then build the part list with pure spreads.
// ---------------------------------------------------------------------------
interface DecodedToolCall {
readonly rawId: string;
readonly name: string;
readonly arguments: unknown;
}
interface DecodedResponse {
readonly text: string | undefined;
readonly reasoning: string | undefined;
readonly toolCalls: ReadonlyArray<DecodedToolCall>;
readonly finishReason: string | undefined;
}
const decodeResponse = (raw: Record<string, unknown>): DecodedResponse => {
const choice = (
raw.choices as
| Array<{
message?: {
content?: string | null;
reasoning_content?: string;
reasoning?: string;
tool_calls?: ReadonlyArray<Record<string, unknown>>;
};
finish_reason?: string;
}>
| undefined
)?.[0];
const message = choice?.message;
const openAiText = message?.content;
const text =
typeof openAiText === "string" && openAiText.length > 0
? openAiText
: nativeTextOf(raw.response);
const reasoning = message?.reasoning_content ?? message?.reasoning;
const rawToolCalls =
message?.tool_calls ??
(Array.isArray(raw.tool_calls)
? (raw.tool_calls as ReadonlyArray<Record<string, unknown>>)
: []);
return {
text,
reasoning: reasoning && reasoning.length > 0 ? reasoning : undefined,
toolCalls: rawToolCalls.flatMap(decodeToolCall),
finishReason: choice?.finish_reason ?? (raw.finish_reason as string | undefined),
};
};
const nativeTextOf = (raw: unknown): string | undefined => {
if (raw == null) return undefined;
// Workers AI responses are JSON-decoded, so JSON.stringify matches
// String() for every scalar and serializes objects/arrays.
const text = typeof raw === "string" ? raw : JSON.stringify(raw);
return text.length > 0 ? text : undefined;
};
const decodeToolCall = (tc: Record<string, unknown>): ReadonlyArray<DecodedToolCall> => {
const fn = tc.function as { name?: string; arguments?: unknown } | undefined;
const rawId = (tc.id as string | undefined) ?? "";
if (fn?.name) {
return [{ rawId, name: fn.name, arguments: fn.arguments ?? "" }];
}
const flatName = tc.name as string | undefined;
if (flatName) {
return [{ rawId, name: flatName, arguments: tc.arguments ?? "" }];
}
return [];
};
const tryParseJsonArgs = (raw: unknown): unknown => {
if (typeof raw !== "string") return raw;
try {
return JSON.parse(raw);
} catch {
// Leave as raw string; the framework's tool-result decoder will fail loudly.
return raw;
}
};
const parseGenerateText = Effect.fn(function* (
raw: Record<string, unknown>,
startsInThink: boolean,
) {
const idGen = yield* IdGenerator.IdGenerator;
const decoded = decodeResponse(raw);
const inline =
decoded.text === undefined ? undefined : splitThinkText(decoded.text, startsInThink);
const reasoning = [decoded.reasoning, inline?.reasoning]
.filter((r): r is string => r !== undefined && r.length > 0)
.join("");
const text = inline?.text ?? decoded.text;
const toolCallParts = yield* Effect.forEach(decoded.toolCalls, (tc) =>
Effect.gen(function* () {
const id = tc.rawId || (yield* idGen.generateId());
return {
type: "tool-call" as const,
id,
name: tc.name,
params: tryParseJsonArgs(tc.arguments),
};
}),
);
const finish = mapFinishReason(
decoded.finishReason ?? (decoded.toolCalls.length > 0 ? "tool_calls" : "stop"),
);
return [
...(reasoning.length > 0 ? [{ type: "reasoning" as const, text: reasoning }] : []),
...(text !== undefined && text.length > 0 ? [{ type: "text" as const, text }] : []),
...toolCallParts,
{
type: "finish" as const,
reason: finish,
usage: mapUsage(raw),
response: undefined,
...finishMetadata(raw),
},
] satisfies ReadonlyArray<Response.PartEncoded>;
});
// ---------------------------------------------------------------------------
// streamText: SSE byte stream → Stream<Response.StreamPartEncoded>
//
// Immutable `StreamState` is threaded through `Stream.mapAccumEffect`. The
// per-chunk output buffer (`parts: Array<StreamPartEncoded>`) is mutable for
// performance — it's scoped to one chunk, never escapes the handler, and lets
// us avoid the O(n²) array-spread that pure threading would force in the hot
// path. This matches the pattern Effect's own `@effect/ai-*` adapters use.
// ---------------------------------------------------------------------------
interface StreamState {
readonly textId: string | undefined;
readonly reasoningId: string | undefined;
readonly toolCalls: ReadonlyMap<
number,
{ readonly id: string; readonly name: string; readonly arguments: string }
>;
readonly lastToolIndex: number | undefined;
readonly closedToolIndices: ReadonlySet<number>;
readonly usage: Record<string, unknown> | undefined;
readonly finishReason: string | undefined;
readonly receivedAnyData: boolean;
readonly receivedDone: boolean;
// Workers AI's *native* streaming for function-calling models (e.g.
// `@cf/meta/llama-3.3-70b`) does not emit structured `tool_calls` deltas
// — it streams the call as a JSON document inside the `response` text
// field (`{"name":"get_weather","parameters":{...}}`). When tools were
// requested we buffer that text here instead of emitting it as
// text-delta, then parse it into tool-params parts on finalize.
readonly nativeToolBuffer: string;
readonly nativeToolId: string | undefined;
readonly think: ThinkState;
}
const initialStreamState = (startsInThink: boolean): StreamState => ({
textId: undefined,
reasoningId: undefined,
toolCalls: new Map(),
lastToolIndex: undefined,
closedToolIndices: new Set(),
usage: undefined,
finishReason: undefined,
receivedAnyData: false,
receivedDone: false,
nativeToolBuffer: "",
nativeToolId: undefined,
think: initialThinkState(startsInThink),
});
type StreamParts = Array<Response.StreamPartEncoded>;
const tryParseJson = (data: string): Record<string, unknown> | undefined => {
try {
const v = JSON.parse(data);
return v && typeof v === "object" ? (v as Record<string, unknown>) : undefined;
} catch {
return undefined;
}
};
const isNullFinalizationToolCall = (tc: Record<string, unknown>): boolean => {
const fn = tc.function as Record<string, unknown> | undefined;
const name = fn?.name ?? tc.name ?? null;
const args = fn?.arguments ?? tc.arguments ?? null;
const id = tc.id ?? null;
return !id && !name && (!args || args === "");
};
const closeReasoning = (state: StreamState, parts: StreamParts): StreamState => {
if (state.reasoningId === undefined) return state;
parts.push({ type: "reasoning-end", id: state.reasoningId });
return { ...state, reasoningId: undefined };
};
const closeToolCall = (state: StreamState, index: number, parts: StreamParts): StreamState => {
if (state.closedToolIndices.has(index)) return state;
const tc = state.toolCalls.get(index);
if (!tc) return state;
parts.push({ type: "tool-params-end", id: tc.id });
parts.push({
type: "tool-call",
id: tc.id,
name: tc.name,
params: tryParseJsonArgs(tc.arguments),
});
const closed = new Set(state.closedToolIndices);
closed.add(index);
return { ...state, closedToolIndices: closed };
};
const emitTextDelta = (
state: StreamState,
delta: string,
parts: StreamParts,
idGen: IdGenerator.Service,
): Effect.Effect<StreamState> =>
Effect.gen(function* () {
let s = closeReasoning(state, parts);
if (s.textId === undefined) {
const id = yield* idGen.generateId();
parts.push({ type: "text-start", id });
s = { ...s, textId: id };
}
parts.push({ type: "text-delta", id: s.textId!, delta });
return s;
});
const emitReasoningDelta = (
state: StreamState,
delta: string,
parts: StreamParts,
idGen: IdGenerator.Service,
): Effect.Effect<StreamState> =>
Effect.gen(function* () {
let s = state;
if (s.reasoningId === undefined) {
const id = yield* idGen.generateId();
parts.push({ type: "reasoning-start", id });
s = { ...s, reasoningId: id };
}
parts.push({ type: "reasoning-delta", id: s.reasoningId!, delta });
return s;
});
const handleToolDeltas = (
state: StreamState,
deltas: ReadonlyArray<Record<string, unknown>>,
parts: StreamParts,
idGen: IdGenerator.Service,
): Effect.Effect<StreamState> =>
Effect.gen(function* () {
let s = state;
for (const d of deltas) {
if (isNullFinalizationToolCall(d)) {
if (s.lastToolIndex !== undefined) {
s = closeToolCall(s, s.lastToolIndex, parts);
}
continue;
}
const idx = (d.index as number | undefined) ?? 0;
const fn = d.function as { name?: string; arguments?: string } | undefined;
const name = fn?.name ?? (d.name as string | undefined) ?? "";
const args = fn?.arguments ?? (d.arguments as string | undefined) ?? "";
const rawId = (d.id as string | undefined) ?? "";
const existing = s.toolCalls.get(idx);
if (existing === undefined) {
if (s.lastToolIndex !== undefined && s.lastToolIndex !== idx) {
s = closeToolCall(s, s.lastToolIndex, parts);
}
const id = rawId || (yield* idGen.generateId());
const entry = { id, name, arguments: args };
const next = new Map(s.toolCalls);
next.set(idx, entry);
s = { ...s, toolCalls: next, lastToolIndex: idx };
parts.push({ type: "tool-params-start", id, name });
if (args.length > 0) {
parts.push({ type: "tool-params-delta", id, delta: args });
}
} else {
// OpenAI-compatible providers stream argument fragments, while
// Workers AI can resend the complete accumulated JSON on every
// chunk. Normalize both shapes to incremental deltas.
const delta = args.startsWith(existing.arguments)
? args.slice(existing.arguments.length)
: args;
const accumulated = args.startsWith(existing.arguments) ? args : existing.arguments + args;
const next = new Map(s.toolCalls);
next.set(idx, { ...existing, arguments: accumulated });
s = { ...s, toolCalls: next, lastToolIndex: idx };
if (delta.length > 0) {
parts.push({
type: "tool-params-delta",
id: existing.id,
delta,
});
}
}
}
return s;
});
const hasNonZeroUsage = (raw: unknown): boolean => {
if (raw == null || typeof raw !== "object") return false;
const u = raw as Record<string, unknown>;
const prompt = (u.prompt_tokens as number | undefined) ?? 0;
const completion = (u.completion_tokens as number | undefined) ?? 0;
const total = (u.total_tokens as number | undefined) ?? 0;
return prompt > 0 || completion > 0 || total > 0;
};
const updateChunkMeta = (state: StreamState, chunk: Record<string, unknown>): StreamState => {
let s = state;
// Workers AI's native stream emits the real usage chunk, then a
// "zero-valued terminator" chunk where every count is 0 (it also re-emits
// `usage` with all zeros). Treat the zero chunk as a no-op so we keep the
// meaningful counts.
if (chunk.usage !== undefined && hasNonZeroUsage(chunk.usage)) {
s = { ...s, usage: chunk };
}
const choices = chunk.choices as Array<{ finish_reason?: string }> | undefined;
const finish = choices?.[0]?.finish_reason ?? (chunk.finish_reason as string | undefined);
if (finish != null) s = { ...s, finishReason: finish };
return s;
};
/**
* Route a text fragment through the inline-reasoning splitter, emitting any
* reasoning it contains and returning the remaining answer text.
*/
const splitInlineReasoning = (
state: StreamState,
input: string,
parts: StreamParts,
idGen: IdGenerator.Service,
): Effect.Effect<readonly [StreamState, string]> =>
Effect.gen(function* () {
const split = splitThink(state.think, input);
let s: StreamState = { ...state, think: split.state };
if (split.reasoning.length > 0) {
s = yield* emitReasoningDelta(s, split.reasoning, parts, idGen);
}
return [s, split.text] as const;
});
const handleNativeText = (
state: StreamState,
chunk: Record<string, unknown>,
parts: StreamParts,
idGen: IdGenerator.Service,
hasTools: boolean,
): Effect.Effect<StreamState> => {
const native = chunk.response;
if (native == null || native === "") return Effect.succeed(state);
const text = typeof native === "string" ? native : JSON.stringify(native);
if (text.length === 0) return Effect.succeed(state);
// Some models (e.g. `@cf/meta/llama-3.3-70b`) mirror every text fragment
// in both the native `response` field and the OpenAI-compatible delta of
// the same chunk. Prefer the latter so the text is neither emitted nor
// buffered twice.
const openAiContent = (chunk.choices as Array<{ delta?: { content?: unknown } }> | undefined)?.[0]
?.delta?.content;
if (openAiContent === text) return Effect.succeed(state);
return Effect.gen(function* () {
const [s, answer] = yield* splitInlineReasoning(state, text, parts, idGen);
if (answer.length === 0) return s;
// When tools were requested, the native `response` stream is the
// tool-call JSON, not prose — buffer it and decide on finalize. We
// pre-allocate the tool id here (we're in an Effect, finalize is sync).
if (hasTools) {
const id = s.nativeToolId ?? (yield* idGen.generateId());
return {
...s,
nativeToolId: id,
nativeToolBuffer: s.nativeToolBuffer + answer,
};
}
return yield* emitTextDelta(s, answer, parts, idGen);
});
};
const handleNativeToolCalls = (
state: StreamState,
chunk: Record<string, unknown>,
parts: StreamParts,
idGen: IdGenerator.Service,
): Effect.Effect<StreamState> => {
const openAiToolCalls = (
chunk.choices as Array<{ delta?: { tool_calls?: ReadonlyArray<unknown> } }> | undefined
)?.[0]?.delta?.tool_calls;
// Workers AI can mirror the same fragments in both its top-level native
// field and the OpenAI-compatible delta. Prefer the latter so every
// fragment is emitted exactly once.
if (Array.isArray(openAiToolCalls)) return Effect.succeed(state);
// Workers AI attaches an empty `tool_calls: []` to ordinary chunks;
// treating it as a tool delta would close the open reasoning block on
// every chunk.
if (!Array.isArray(chunk.tool_calls) || chunk.tool_calls.length === 0) {
return Effect.succeed(state);
}
return Effect.gen(function* () {
const s = closeReasoning(state, parts);
return yield* handleToolDeltas(
s,
chunk.tool_calls as ReadonlyArray<Record<string, unknown>>,
parts,
idGen,
);
});
};
const handleOpenAiDelta = (
state: StreamState,
chunk: Record<string, unknown>,
parts: StreamParts,
idGen: IdGenerator.Service,
): Effect.Effect<StreamState> => {
const delta = (chunk.choices as Array<{ delta?: Record<string, unknown> }> | undefined)?.[0]
?.delta;
if (!delta) return Effect.succeed(state);
return Effect.gen(function* () {
let s = state;
const reasoning = (delta.reasoning_content ?? delta.reasoning) as string | undefined;
if (reasoning && reasoning.length > 0) {
s = yield* emitReasoningDelta(s, reasoning, parts, idGen);
}
const text = delta.content as string | undefined;
if (text && text.length > 0) {
const [next, answer] = yield* splitInlineReasoning(s, text, parts, idGen);
s = answer.length > 0 ? yield* emitTextDelta(next, answer, parts, idGen) : next;
}
const toolDeltas = delta.tool_calls as ReadonlyArray<Record<string, unknown>> | undefined;
if (Array.isArray(toolDeltas) && toolDeltas.length > 0) {
s = closeReasoning(s, parts);
s = yield* handleToolDeltas(s, toolDeltas, parts, idGen);
}
return s;
});
};
const handleStreamChunk = (
state: StreamState,
data: string,
idGen: IdGenerator.Service,
hasTools: boolean,
): Effect.Effect<readonly [StreamState, ReadonlyArray<Response.StreamPartEncoded>]> =>
Effect.gen(function* () {
if (data === "") return [state, []] as const;
if (data === "[DONE]") {
return [{ ...state, receivedDone: true }, []] as const;
}
const chunk = tryParseJson(data);
if (chunk === undefined) return [state, []] as const;
const parts: StreamParts = [];
let s: StreamState = { ...state, receivedAnyData: true };
s = updateChunkMeta(s, chunk);
s = yield* handleNativeText(s, chunk, parts, idGen, hasTools);
s = yield* handleNativeToolCalls(s, chunk, parts, idGen);
s = yield* handleOpenAiDelta(s, chunk, parts, idGen);
return [s, parts] as const;
});
/**
* Decode a buffered native-streaming tool call. Workers AI's native shape
* for a function call is a JSON document with `name` plus `parameters` (or
* `arguments`), optionally wrapped in an array. Returns one entry per call,
* or `undefined` if the buffer isn't a tool-call document (so the caller
* can fall back to treating it as plain text).
*/
const decodeNativeToolBuffer = (
buffer: string,
): ReadonlyArray<{ name: string; args: string }> | undefined => {
const trimmed = buffer.trim();
if (trimmed.length === 0 || !(trimmed[0] === "{" || trimmed[0] === "[")) {
return undefined;
}
let parsed: unknown;
try {
parsed = JSON.parse(trimmed);
} catch {
return undefined;
}
const items = Array.isArray(parsed) ? parsed : [parsed];
const calls = items.flatMap(
(
item,
): ReadonlyArray<{
name: string;
args: string;
}> => {
if (item == null || typeof item !== "object") return [];
const rec = item as Record<string, unknown>;
const fn = rec.function as Record<string, unknown> | undefined;
const name = (fn?.name ?? rec.name) as string | undefined;
if (typeof name !== "string" || name.length === 0) return [];
const rawArgs = fn?.arguments ?? rec.arguments ?? rec.parameters ?? {};
const args = typeof rawArgs === "string" ? rawArgs : JSON.stringify(rawArgs);
return [{ name, args }];
},
);
return calls.length > 0 ? calls : undefined;
};
const flushNativeToolBuffer = (state: StreamState, parts: StreamParts): StreamState => {
if (state.nativeToolBuffer.length === 0) return state;
const calls = decodeNativeToolBuffer(state.nativeToolBuffer);
// Not a tool-call document — the model returned prose despite tools being
// available; surface it as text so nothing is dropped.
if (calls === undefined) {
const id = state.nativeToolId ?? "text-0";
parts.push({ type: "text-start", id });
parts.push({ type: "text-delta", id, delta: state.nativeToolBuffer });
parts.push({ type: "text-end", id });
return { ...state, nativeToolBuffer: "", nativeToolId: undefined };
}
const baseId = state.nativeToolId ?? "tool-0";
calls.forEach((call, i) => {
const id = i === 0 ? baseId : `${baseId}-${i}`;
parts.push({ type: "tool-params-start", id, name: call.name });
if (call.args.length > 0) {
parts.push({ type: "tool-params-delta", id, delta: call.args });
}
parts.push({ type: "tool-params-end", id });
parts.push({ type: "tool-call", id, name: call.name, params: tryParseJsonArgs(call.args) });
});
return { ...state, nativeToolBuffer: "", nativeToolId: undefined };
};
/**
* Emit text the inline-reasoning splitter held back as a possible split tag
* when the stream ended before it resolved.
*/
const flushThinkCarry = (state: StreamState, parts: StreamParts): StreamState => {
const { mode, carry } = state.think;
let s: StreamState = { ...state, think: { mode, carry: "" } };
if (carry.length === 0) return s;
if (mode === "inside") {
if (s.reasoningId === undefined) {
parts.push({ type: "reasoning-start", id: "reasoning-0" });
s = { ...s, reasoningId: "reasoning-0" };
}
parts.push({ type: "reasoning-delta", id: s.reasoningId!, delta: carry });
return s;
}
if (carry.trim().length === 0) return s;
s = closeReasoning(s, parts);
if (s.textId === undefined) {
parts.push({ type: "text-start", id: "text-0" });
s = { ...s, textId: "text-0" };
}
parts.push({ type: "text-delta", id: s.textId!, delta: carry });
return s;
};
const finalizeStream = (state: StreamState): ReadonlyArray<Response.StreamPartEncoded> => {
const parts: StreamParts = [];
let s = flushThinkCarry(state, parts);
for (const [idx] of s.toolCalls) s = closeToolCall(s, idx, parts);
s = closeReasoning(s, parts);
if (s.textId !== undefined) parts.push({ type: "text-end", id: s.textId });
// Emit any buffered native-streaming tool call (llama-style) before the
// terminal finish part.
s = flushNativeToolBuffer(s, parts);
// Three cases for the final reason:
// 1. The model emitted an explicit `finish_reason` → map it.
// 2. The stream ended cleanly (`[DONE]` seen) but no reason → "stop".
// Workers AI's native shape never includes `finish_reason`,
// so without this rule every native-mode stream would report
// `unknown` despite completing successfully.
// 3. The stream ended abnormally (no `[DONE]`, no reason) → "error".
const reason: Response.FinishReason =
s.finishReason !== undefined
? mapFinishReason(s.finishReason)
: s.receivedDone
? "stop"
: s.receivedAnyData
? "error"
: "unknown";
parts.push({
type: "finish",
reason,
usage: mapUsage(s.usage),
response: undefined,
...finishMetadata(s.usage),
});
return parts;
};
const parseStreamText = (
resp: Response,
idGen: IdGenerator.Service,
hasTools: boolean,
startsInThink: boolean,
): Stream.Stream<Response.StreamPartEncoded, AiError.AiError> => {
const body = resp.body;
if (body === null) {
return Stream.fromIterable<Response.StreamPartEncoded>(
finalizeStream(initialStreamState(startsInThink)),
);
}
return Stream.fromReadableStream<Uint8Array, AiError.AiError>({
evaluate: () => body,
onError: (cause) => toAiError(cause, "streamText"),
}).pipe(
Stream.decodeText(),
Stream.pipeThroughChannel(Sse.decode<AiError.AiError, unknown>()),
Stream.catchTags({
Retry: (retry) => Stream.die(retry),
SseError: (error) => Stream.fail(toAiError(error, "streamText")),
}),
Stream.mapAccumEffect(
() => initialStreamState(startsInThink),
(state, event) => handleStreamChunk(state, event.data, idGen, hasTools),
{ onHalt: (state) => finalizeStream(state) },
),
);
};
// ---------------------------------------------------------------------------
// Error mapping
// ---------------------------------------------------------------------------
const toAiError = (cause: unknown, method: "generateText" | "streamText"): AiError.AiError =>
AiError.AiError.make({
module: "Cloudflare.AI.LanguageModel",
method,
reason: AiError.UnknownError.make({
description: cause instanceof Error ? cause.message : "AI Gateway request failed",
}),
});