sailfork / t3code-github / .repos/ alchemy-effect/ packages/ cloudflare/ src/ Pipelines/ StreamCodec.ts

import type { RuntimeContext } from "alchemy/RuntimeContext";
import type { Pipeline, PipelineRecord } from "cloudflare:pipelines";
import * as Effect from "effect/Effect";
import * as Schema from "effect/Schema";
import { isLegacyPipeline, type LegacyPipeline } from "./LegacyPipeline.ts";
import type { Stream } from "./Stream.ts";
import { StreamSendError, type WriteStreamClient } from "./WriteStream.ts";

/**
 * Shared record encoding + client assembly for the WriteStream / StreamSink
 * implementations. Internal — not exported from the Pipelines barrel.
 */

/**
 * The Effect Schema a stream was declared with, if any. Legacy pipelines
 * have none.
 */
export const recordSchemaOf = (stream: Stream<any> | LegacyPipeline): Schema.Top | undefined => {
  if (isLegacyPipeline(stream)) return undefined;
  // References (`Stream.ref`) proxy unknown properties to Outputs — only a
  // real Effect Schema counts.
  const schema: unknown = stream.RecordSchema;
  return Schema.isSchema(schema) ? schema : undefined;
};

/**
 * Build the batch encoder: `Schema.toCodecJson(schema)` per record when
 * the stream has an Effect Schema, identity otherwise.
 */
export const makeRecordEncoder = (
  schema: Schema.Top | undefined,
): ((
  records: ReadonlyArray<unknown>,
) => Effect.Effect<ReadonlyArray<PipelineRecord>, StreamSendError>) => {
  if (schema === undefined) {
    return (records) => Effect.succeed(records as ReadonlyArray<PipelineRecord>);
  }
  const encode = Schema.encodeUnknownEffect(
    Schema.toCodecJson(schema as unknown as Schema.Codec<unknown, unknown, never, never>),
  );
  return (records) =>
    Effect.forEach(records, (record, index) =>
      encode(record).pipe(
        Effect.map((json) => json as PipelineRecord),
        Effect.mapError(
          (issue) =>
            new StreamSendError({
              message: `record ${index} does not match the stream schema: ${issue.message}`,
              cause: issue,
              reason: "InvalidRecord",
              index,
            }),
        ),
      ),
    );
};

/** Wrap any transport failure as a `SendFailed` {@link StreamSendError}. */
export const toStreamSendError = (error: unknown): StreamSendError =>
  new StreamSendError({
    message:
      typeof error === "object" && error !== null && "message" in error
        ? String((error as { message: unknown }).message)
        : "Unknown pipeline error",
    cause: error,
    reason: "SendFailed",
  });

/** Assemble a {@link WriteStreamClient} from an encoder and a transport. */
export const makeStreamClient = (options: {
  schema: Schema.Top | undefined;
  raw: Effect.Effect<Pipeline, never, RuntimeContext>;
  sendEncoded: (
    records: ReadonlyArray<PipelineRecord>,
  ) => Effect.Effect<void, StreamSendError, RuntimeContext>;
}): WriteStreamClient<any> => {
  const encode = makeRecordEncoder(options.schema);
  return {
    raw: options.raw,
    encode,
    sendEncoded: options.sendEncoded,
    send: (records) => encode(records).pipe(Effect.flatMap(options.sendEncoded)),
  };
};

const encoder = new TextEncoder();

/**
 * Approximate wire size of one encoded record in the JSON array body:
 * its UTF-8 JSON plus the separating comma. Values JSON cannot encode
 * count as 1 byte; the send rejects them and the error surfaces.
 */
export const recordSize = (record: PipelineRecord): number => {
  let json: string | undefined;
  try {
    json = JSON.stringify(record);
  } catch {
    json = undefined;
  }
  return (json === undefined ? 0 : encoder.encode(json).length) + 1;
};