sailfork / t3code-github / .repos/ alchemy-effect/ packages/ neon/ src/ LanguageModel.ts

import type { RuntimeContext } from "alchemy/RuntimeContext";
import { AiError, LanguageModel, Response, Tool } from "effect/ai";
import { toCodecOpenAI } from "effect/ai/OpenAiStructuredOutput";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Sse from "effect/encoding/Sse";
import * as FetchHttpClient from "effect/http/FetchHttpClient";
import * as HttpClient from "effect/http/HttpClient";
import * as HttpClientRequest from "effect/http/HttpClientRequest";
import type * as HttpClientResponse from "effect/http/HttpClientResponse";
import * as Layer from "effect/Layer";
import type * as Redacted from "effect/Redacted";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";

/** Runtime-only connection obtained from QueryAIGateway, never an account API key. */
export interface LanguageModelClient {
  /** OpenAI-compatible base URL ending in /v1. */
  readonly chatBaseUrl: Effect.Effect<string, never, RuntimeContext>;
  /** Redacted credential with ai_gateway:invoke scope. */
  readonly token: Effect.Effect<Redacted.Redacted<string>, never, RuntimeContext>;
}

/** Configuration for Neon's OpenAI-compatible Chat Completions endpoint. */
export interface LanguageModelOptions {
  /** Model ID from Neon's catalog that supports Chat Completions. */
  readonly model: string;
  /** Model-dependent generation settings, sent without silently rewriting them. */
  readonly parameters?: {
    /** Sampling temperature. */
    readonly temperature?: number;
    /** Maximum output tokens; mutually exclusive with maxCompletionTokens. */
    readonly maxTokens?: number;
    /** Completion budget for models using max_completion_tokens. */
    readonly maxCompletionTokens?: number;
    /** Nucleus sampling probability. */
    readonly topP?: number;
    /** Provider-supported deterministic sampling seed. */
    readonly seed?: number;
    /** Repetition penalty based on token frequency. */
    readonly frequencyPenalty?: number;
    /** Repetition penalty based on token presence. */
    readonly presencePenalty?: number;
  };
}

/**
 * Effect AI over Neon's documented `/v1/chat/completions` route. Supports text,
 * image inputs, function tools, JSON-schema output and cancellable SSE streams.
 * Tool and structured-output support depends on the selected model. Responses-only
 * models (including Codex), native Anthropic thinking/cache controls, audio,
 * provider-executed tools and image generation are not supported by this adapter.
 * Use the bound client's dialect URLs with a native SDK for those APIs.
 *
 * ### Generate text
 * **Example:** A request-scoped Effect model
 * ```typescript
 * const ai = yield* Neon.QueryAIGateway(gateway);
 * const model = ai.model({ model: "gpt-5-mini" });
 * // Inside the Function or Worker request handler:
 * const reply = yield* LanguageModel.generateText({ prompt: "Say hello." }).pipe(
 *   Effect.provide(model),
 * );
 * ```
 *
 * ### Generate structured output
 * **Example:** Decode the response with an Effect Schema
 * ```typescript
 * const reply = yield* LanguageModel.generateObject({
 *   prompt: "Return a short greeting.",
 *   schema: Schema.Struct({ greeting: Schema.String }),
 * }).pipe(Effect.provide(ai.model({ model: "gpt-5-mini" })));
 * ```
 *
 * @layer
 * @provides effect/ai/LanguageModel
 * @product AI Gateway
 * @category AI Gateway
 */
export const makeLanguageModelLayer = (
  options: LanguageModelOptions & { readonly client: LanguageModelClient },
): Layer.Layer<LanguageModel.LanguageModel, never, RuntimeContext> =>
  Layer.effect(LanguageModel.LanguageModel, makeLanguageModel(options)).pipe(
    Layer.provide(FetchHttpClient.layer),
  );

/** Construct with an injected Effect HTTP client for custom transports and protocol tests. */
export const makeLanguageModel = ({
  client,
  model,
  parameters,
}: LanguageModelOptions & {
  readonly client: LanguageModelClient;
}): Effect.Effect<LanguageModel.LanguageModel, never, RuntimeContext | HttpClient.HttpClient> =>
  Effect.gen(function* () {
    const http = yield* HttpClient.HttpClient;
    const baseUrl = yield* client.chatBaseUrl;
    const token = yield* client.token;
    const send = (options: LanguageModel.ProviderOptions, stream: boolean) =>
      Effect.gen(function* () {
        const body = yield* requestBody(options, model, parameters, stream);
        const request = yield* HttpClientRequest.bodyJson(body)(
          HttpClientRequest.post(`${baseUrl.replace(/\/$/, "")}/chat/completions`).pipe(
            HttpClientRequest.bearerToken(token),
            HttpClientRequest.accept(stream ? "text/event-stream" : "application/json"),
          ),
        ).pipe(Effect.mapError(() => invalidRequest("Request is not JSON serializable")));
        const response = yield* http.execute(request).pipe(
          Effect.mapError(() =>
            error(
              AiError.NetworkError.make({
                reason: "TransportError",
                method: "POST",
                url: "/v1/chat/completions",
                description: "Neon AI Gateway transport failed",
              }),
            ),
          ),
        );
        if (response.status < 200 || response.status >= 300) {
          return yield* httpError(response);
        }
        return response;
      });
    return yield* LanguageModel.make({
      codecTransformer: toCodecOpenAI,
      generateText: (options) =>
        Effect.gen(function* () {
          const response = yield* send(options, false);
          const raw = yield* response.json.pipe(
            Effect.mapError(() => invalidOutput("Expected a JSON completion")),
          );
          const decoded = yield* decode(Completion, raw);
          const choice = decoded.choices[0];
          if (!choice) return yield* invalidOutput("Completion has no choices");
          const parts: Array<Response.PartEncoded> = [];
          const message = choice.message;
          if (message.refusal)
            return yield* error(
              AiError.ContentPolicyError.make({
                description: "Model refused the request",
              }),
            );
          if (message.reasoning_content)
            parts.push({ type: "reasoning", text: message.reasoning_content });
          if (typeof message.content === "string") {
            if (message.content) parts.push({ type: "text", text: message.content });
          } else if (message.content) {
            for (const block of message.content) parts.push({ type: "text", text: block.text });
          }
          for (const call of message.tool_calls ?? []) {
            parts.push({
              type: "tool-call",
              id: call.id,
              name: call.function.name,
              params: yield* parseArguments(call.function.arguments),
            });
          }
          parts.push({
            type: "finish",
            reason: finishReason(choice.finish_reason),
            usage: usage(decoded.usage),
          });
          return parts;
        }),
      streamText: (options) =>
        Stream.unwrap(
          Effect.gen(function* () {
            const response = yield* send(options, true);
            return completionStream(response);
          }),
        ),
    });
  });

const error = (reason: AiError.AiError["reason"]) =>
  AiError.make({
    module: "Neon.LanguageModel",
    method: "chatCompletions",
    reason,
  });
const invalidOutput = (description: string) =>
  error(AiError.InvalidOutputError.make({ description }));
const invalidRequest = (description: string) =>
  error(AiError.InvalidRequestError.make({ description }));
const decode = <S extends Schema.ConstraintDecoder<unknown>>(schema: S, value: unknown) =>
  Schema.decodeUnknownEffect(schema)(value).pipe(
    Effect.mapError(() => invalidOutput("Invalid Chat Completions response shape")),
  );
const parseArguments = (value: string) =>
  Schema.decodeEffect(Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)))(
    value,
  ).pipe(Effect.mapError(() => invalidOutput("Invalid function-call JSON arguments")));

const TokenCount = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0));
const Usage = Schema.Struct({
  prompt_tokens: Schema.optional(TokenCount),
  completion_tokens: Schema.optional(TokenCount),
  prompt_tokens_details: Schema.Struct({ cached_tokens: Schema.optional(TokenCount) }).pipe(
    Schema.NullOr,
    Schema.optional,
  ),
  completion_tokens_details: Schema.Struct({ reasoning_tokens: Schema.optional(TokenCount) }).pipe(
    Schema.NullOr,
    Schema.optional,
  ),
});
const Content = Schema.NullOr(
  Schema.Union([
    Schema.String,
    Schema.Array(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })),
  ]),
);
const Completion = Schema.Struct({
  choices: Schema.Array(
    Schema.Struct({
      finish_reason: Schema.NullOr(Schema.String),
      message: Schema.Struct({
        content: Schema.optional(Content),
        reasoning_content: Schema.String.pipe(Schema.NullOr, Schema.optional),
        refusal: Schema.String.pipe(Schema.NullOr, Schema.optional),
        tool_calls: Schema.Struct({
          id: Schema.String,
          type: Schema.Literal("function"),
          function: Schema.Struct({
            name: Schema.String,
            arguments: Schema.String,
          }),
        }).pipe(Schema.Array, Schema.optional),
      }),
    }),
  ),
  usage: Usage.pipe(Schema.NullOr, Schema.optional),
});
const Chunk = Schema.Struct({
  choices: Schema.Array(
    Schema.Struct({
      index: TokenCount,
      finish_reason: Schema.String.pipe(Schema.NullOr, Schema.optional),
      delta: Schema.Struct({
        content: Schema.String.pipe(Schema.NullOr, Schema.optional),
        reasoning_content: Schema.String.pipe(Schema.NullOr, Schema.optional),
        refusal: Schema.String.pipe(Schema.NullOr, Schema.optional),
        tool_calls: Schema.Struct({
          index: TokenCount,
          id: Schema.optional(Schema.String),
          type: Schema.optional(Schema.Literal("function")),
          function: Schema.optional(
            Schema.Struct({
              name: Schema.optional(Schema.String),
              arguments: Schema.optional(Schema.String),
            }),
          ),
        }).pipe(Schema.Array, Schema.optional),
      }),
    }),
  ),
  usage: Usage.pipe(Schema.NullOr, Schema.optional),
});
const ErrorBody = Schema.Struct({
  error: Schema.Struct({
    code: Schema.optional(Schema.Union([Schema.String, Schema.Number, Schema.Null])),
    message: Schema.optional(Schema.String),
  }),
});

const httpError = (
  response: HttpClientResponse.HttpClientResponse,
): Effect.Effect<never, AiError.AiError> =>
  Effect.gen(function* () {
    // Never retain upstream bodies, headers or request objects in public errors.
    const body = yield* response.json.pipe(
      Effect.flatMap(Schema.decodeUnknownEffect(ErrorBody)),
      Effect.orElseSucceed(() => undefined),
    );
    const status = response.status;
    if (status === 403 && body?.error.message === "ai gateway not enabled for account")
      return yield* error(AiError.QuotaExhaustedError.make({}));
    if (status === 401 || status === 403)
      return yield* error(
        AiError.AuthenticationError.make({
          kind: status === 401 ? "InvalidKey" : "InsufficientPermissions",
        }),
      );
    if (
      status === 402 ||
      body?.error.code === "REQUEST_LIMIT_EXCEEDED" ||
      body?.error.code === "insufficient_quota"
    )
      return yield* error(AiError.QuotaExhaustedError.make({}));
    if (status === 429) {
      const seconds = Number(response.headers["retry-after"]);
      return yield* error(
        AiError.RateLimitError.make(
          Number.isFinite(seconds) && seconds >= 0 ? { retryAfter: Duration.seconds(seconds) } : {},
        ),
      );
    }
    if (status >= 500)
      return yield* error(
        AiError.InternalProviderError.make({
          description: `Neon AI Gateway returned HTTP ${status}`,
        }),
      );
    return yield* invalidRequest(
      `Neon AI Gateway rejected the request (HTTP ${status}); check the model, endpoint and parameters`,
    );
  });

const finishReason = (reason: string | null | undefined): Response.FinishReason => {
  switch (reason) {
    case "stop":
      return "stop";
    case "length":
      return "length";
    case "tool_calls":
      return "tool-calls";
    case "content_filter":
      return "content-filter";
    case null:
    case undefined:
      return "unknown";
    default:
      return "other";
  }
};
const usage = (value: typeof Usage.Type | null | undefined) => {
  const input = value?.prompt_tokens;
  const cached = value?.prompt_tokens_details?.cached_tokens;
  const output = value?.completion_tokens;
  const reasoning = value?.completion_tokens_details?.reasoning_tokens;
  return Response.Usage.make({
    inputTokens: {
      total: input,
      uncached: input === undefined ? undefined : Math.max(0, input - (cached ?? 0)),
      cacheRead: cached,
      cacheWrite: undefined,
    },
    outputTokens: {
      total: output,
      text: output === undefined ? undefined : Math.max(0, output - (reasoning ?? 0)),
      reasoning,
    },
  });
};

const requestBody = (
  options: LanguageModel.ProviderOptions,
  model: string,
  parameters: LanguageModelOptions["parameters"],
  stream: boolean,
) =>
  Effect.gen(function* () {
    if (parameters?.maxTokens !== undefined && parameters.maxCompletionTokens !== undefined)
      return yield* invalidRequest("Set maxTokens or maxCompletionTokens, not both");
    if (options.tools.some(Tool.isProviderDefined))
      return yield* invalidRequest("Provider-executed tools require a native provider API");
    const messages: Array<Record<string, unknown>> = [];
    for (const message of options.prompt.content) {
      if (message.role === "system") {
        messages.push({ role: "system", content: message.content });
      } else if (message.role === "user") {
        const content: Array<unknown> = [];
        for (const part of message.content) {
          if (part.type === "text") content.push({ type: "text", text: part.text });
          else if (part.type === "file" && part.mediaType.startsWith("image/")) {
            const url = yield* Effect.sync(() => {
              if (part.data instanceof URL) return part.data.toString();
              if (part.data instanceof Uint8Array) {
                let binary = "";
                for (const byte of part.data) binary += String.fromCharCode(byte);
                return `data:${part.mediaType};base64,${btoa(binary)}`;
              }
              return /^(data:|https?:)/.test(part.data)
                ? part.data
                : `data:${part.mediaType};base64,${part.data}`;
            });
            content.push({ type: "image_url", image_url: { url } });
          } else return yield* invalidRequest("Only text and image user inputs are supported");
        }
        messages.push({ role: "user", content });
      } else if (message.role === "assistant") {
        const text: Array<string> = [];
        const toolCalls: Array<unknown> = [];
        for (const part of message.content) {
          if (part.type === "text") text.push(part.text);
          else if (part.type === "tool-call")
            toolCalls.push({
              id: part.id,
              type: "function",
              function: {
                name: part.name,
                arguments: yield* stringify(part.params),
              },
            });
          else if (part.type !== "reasoning")
            return yield* invalidRequest("Unsupported assistant content for Chat Completions");
        }
        messages.push({
          role: "assistant",
          content: text.join("") || null,
          ...(toolCalls.length ? { tool_calls: toolCalls } : {}),
        });
      } else {
        for (const part of message.content) {
          if (part.type !== "tool-result")
            return yield* invalidRequest("Unsupported tool message for Chat Completions");
          messages.push({
            role: "tool",
            tool_call_id: part.id,
            content: typeof part.result === "string" ? part.result : yield* stringify(part.result),
          });
        }
      }
    }
    const choice = options.toolChoice;
    const selected =
      typeof choice === "object" && "oneOf" in choice
        ? options.tools.filter((tool) => choice.oneOf.includes(tool.name))
        : options.tools;
    const tools = yield* Effect.try({
      try: () =>
        selected.map((tool) => ({
          type: "function",
          function: {
            name: tool.name,
            description: Tool.getDescription(tool),
            parameters: Tool.getJsonSchema(tool),
          },
        })),
      catch: () => invalidRequest("Tool schema cannot be represented as JSON Schema"),
    });
    const responseFormat = options.responseFormat;
    const jsonSchema =
      responseFormat.type === "json"
        ? yield* Effect.try({
            try: () => toCodecOpenAI(responseFormat.schema).jsonSchema,
            catch: () =>
              error(
                AiError.UnsupportedSchemaError.make({
                  description: "Schema is not supported by OpenAI structured output",
                }),
              ),
          })
        : undefined;
    return {
      model,
      messages,
      stream,
      ...(stream ? { stream_options: { include_usage: true } } : {}),
      ...(tools.length
        ? {
            tools,
            tool_choice:
              typeof choice === "object"
                ? "tool" in choice
                  ? { type: "function", function: { name: choice.tool } }
                  : (choice.mode ?? "auto")
                : choice,
          }
        : {}),
      ...(jsonSchema
        ? {
            response_format: {
              type: "json_schema",
              json_schema: {
                name: responseFormat.type === "json" ? responseFormat.objectName : "response",
                schema: jsonSchema,
                strict: true,
              },
            },
          }
        : {}),
      temperature: parameters?.temperature,
      max_tokens: parameters?.maxTokens,
      max_completion_tokens: parameters?.maxCompletionTokens,
      top_p: parameters?.topP,
      seed: parameters?.seed,
      frequency_penalty: parameters?.frequencyPenalty,
      presence_penalty: parameters?.presencePenalty,
    };
  });
const stringify = (value: unknown) =>
  Effect.try({
    try: () => JSON.stringify(value),
    catch: () => invalidRequest("Prompt contains a non-JSON value"),
  });

const completionStream = (
  response: HttpClientResponse.HttpClientResponse,
): Stream.Stream<Response.StreamPartEncoded, AiError.AiError> =>
  Stream.unwrap(
    Effect.sync(() => {
      let done = false;
      let reason: string | undefined;
      let tokenUsage: typeof Usage.Type | null | undefined;
      let textStarted = false;
      let reasoningStarted = false;
      const calls = new Map<
        number,
        { id: string; name: string; arguments: string; started: boolean }
      >();
      const chunks = response.stream.pipe(
        Stream.mapError(() => invalidOutput("Chat Completions stream transport failed")),
        Stream.decodeText(),
        Stream.pipeThroughChannel(Sse.decode<AiError.AiError, unknown>()),
        Stream.mapError(() => invalidOutput("Invalid or interrupted SSE stream")),
        Stream.takeUntil((event) => event.data === "[DONE]"),
        Stream.mapEffect((event) =>
          Effect.gen(function* () {
            const parts: Array<Response.StreamPartEncoded> = [];
            if (event.data === "[DONE]") {
              done = true;
              return parts;
            }
            if (!event.data) return parts;
            if (event.event === "error")
              return yield* error(
                AiError.InternalProviderError.make({
                  description: "Neon AI Gateway stream reported an error",
                }),
              );
            const chunk = yield* Schema.decodeEffect(Schema.fromJsonString(Chunk))(event.data).pipe(
              Effect.mapError(() => invalidOutput("Invalid Chat Completions stream chunk")),
            );
            if (chunk.usage != null) tokenUsage = chunk.usage;
            for (const choice of chunk.choices) {
              if (choice.index !== 0) continue;
              if (choice.finish_reason != null) reason = choice.finish_reason;
              const delta = choice.delta;
              if (delta.refusal)
                return yield* error(
                  AiError.ContentPolicyError.make({
                    description: "Model refused the request",
                  }),
                );
              if (delta.reasoning_content) {
                if (!reasoningStarted) {
                  reasoningStarted = true;
                  parts.push({ type: "reasoning-start", id: "reasoning" });
                }
                parts.push({
                  type: "reasoning-delta",
                  id: "reasoning",
                  delta: delta.reasoning_content,
                });
              }
              if (delta.content) {
                if (!textStarted) {
                  textStarted = true;
                  parts.push({ type: "text-start", id: "text" });
                }
                parts.push({
                  type: "text-delta",
                  id: "text",
                  delta: delta.content,
                });
              }
              for (const deltaCall of delta.tool_calls ?? []) {
                const call = calls.get(deltaCall.index) ?? {
                  id: "",
                  name: "",
                  arguments: "",
                  started: false,
                };
                if (deltaCall.id) {
                  if (call.id && call.id !== deltaCall.id)
                    return yield* invalidOutput("Tool call ID changed within a stream");
                  call.id = deltaCall.id;
                }
                if (deltaCall.function?.name) {
                  if (call.started && call.name !== deltaCall.function.name)
                    return yield* invalidOutput("Tool name changed within a stream");
                  call.name = deltaCall.function.name;
                }
                const args = deltaCall.function?.arguments ?? "";
                call.arguments += args;
                if (!call.started && call.id && call.name) {
                  call.started = true;
                  parts.push({
                    type: "tool-params-start",
                    id: call.id,
                    name: call.name,
                  });
                  if (call.arguments)
                    parts.push({
                      type: "tool-params-delta",
                      id: call.id,
                      delta: call.arguments,
                    });
                } else if (call.started && args)
                  parts.push({
                    type: "tool-params-delta",
                    id: call.id,
                    delta: args,
                  });
                calls.set(deltaCall.index, call);
              }
            }
            return parts;
          }),
        ),
        Stream.flatMap(Stream.fromIterable),
      );
      const finish = Stream.unwrap(
        Effect.gen(function* () {
          if (!done || !reason)
            return yield* invalidOutput(
              "Chat Completions stream ended before its finish reason and [DONE]",
            );
          const parts: Array<Response.StreamPartEncoded> = [];
          if (reasoningStarted) parts.push({ type: "reasoning-end", id: "reasoning" });
          if (textStarted) parts.push({ type: "text-end", id: "text" });
          for (const call of calls.values()) {
            if (!call.started) return yield* invalidOutput("Incomplete streamed tool identity");
            parts.push({ type: "tool-params-end", id: call.id });
            if (reason !== "length" && reason !== "content_filter") {
              parts.push({
                type: "tool-call",
                id: call.id,
                name: call.name,
                params: yield* parseArguments(call.arguments),
              });
            }
          }
          parts.push({
            type: "finish",
            reason: finishReason(reason),
            usage: usage(tokenUsage),
          });
          return Stream.fromIterable(parts);
        }),
      );
      return Stream.concat(chunks, finish);
    }),
  );