sailfork / t3code-github / .repos/ alchemy-effect/ packages/ integration-tests/ test/ Cloudflare/ K2/ StreamSink.test.ts

import {
  decodeWithSchema,
  encodeRecords,
  fromBase64,
  fromConsumedRecords,
  toBase64,
  toProduceRecords,
  wireSize,
} from "@alchemy.run/cloudflare/K2/K2Codec";
import type { EncodedRecord } from "@alchemy.run/cloudflare/K2/K2Types";
import {
  makeStreamSink,
  MAX_REQUEST_BYTES,
  packStreamSinkBatches,
} from "@alchemy.run/cloudflare/K2/StreamSinkBatch";
import type { WriteStreamClient } from "@alchemy.run/cloudflare/K2/WriteStream";
import {
  type K2StreamBinding,
  sendViaBinding,
} from "@alchemy.run/cloudflare/K2/WriteStreamBinding";
import { makeWriteStreamClient } from "@alchemy.run/cloudflare/K2/WriteStreamClient";
import { describe, expect, it } from "@alchemy.run/test";
import * as k2 from "@distilled.cloud/cloudflare/k2";
import { RuntimeContext } from "alchemy/RuntimeContext";
import * as Effect from "effect/Effect";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";

/**
 * Pure tests of K2 record encoding, `StreamSink` packing and retries, and
 * the native binding's error mapping. Live delivery is covered by
 * `WriteStream.test.ts` and `ReadSubscription.test.ts`.
 */

const Order = Schema.Struct({ id: Schema.Number });

const record = (bytes: number): EncodedRecord => ({ content: new Uint8Array(bytes) });

const fakeClient = (failures: ReadonlyArray<k2.ProduceError["_tag"]> = []) => {
  const sent: EncodedRecord[][] = [];
  let attempt = 0;
  const client: WriteStreamClient<{ id: number }> = makeWriteStreamClient(Order, (records) =>
    Effect.suspend((): Effect.Effect<void, k2.K2Unavailable | k2.K2AppendOutcomeUnknown> => {
      const tag = failures[attempt++];
      if (tag === "K2Unavailable") {
        return Effect.fail(new k2.K2Unavailable({ code: 10211, message: "unavailable" }));
      }
      if (tag === "K2AppendOutcomeUnknown") {
        return Effect.fail(new k2.K2AppendOutcomeUnknown({ code: 10212, message: "maybe stored" }));
      }
      sent.push([...records]);
      return Effect.void;
    }),
  );
  return { client, sent, attempts: () => attempt };
};

const run = <A>(stream: Stream.Stream<A>, client: WriteStreamClient<A>) =>
  stream.pipe(
    Stream.run(makeStreamSink(client)),
    Effect.provideService(RuntimeContext, {} as never),
  );

describe("Cloudflare K2 records and StreamSink", { tags: ["unit", "local"] }, () => {
  it("round-trips bytes through standard base64", () => {
    const bytes = Uint8Array.from({ length: 70_000 }, (_, i) => i % 256);
    expect(fromBase64(toBase64(bytes))).toEqual(bytes);
    expect(toBase64(new TextEncoder().encode("hi?"))).toEqual("aGk/");
  });

  it("encodes strings as UTF-8 and drops empty headers", () => {
    const [a, b] = encodeRecords([
      { content: "é", headers: {} },
      { content: new ArrayBuffer(2), headers: { k: "v" } },
    ]);
    expect(a!.content).toEqual(new Uint8Array([0xc3, 0xa9]));
    expect(a!.headers).toBeUndefined();
    expect(b!.content.byteLength).toEqual(2);
    expect(toProduceRecords([b!])).toEqual([{ content: "AAA=", headers: { k: "v" } }]);
  });

  it.effect("decodes consumed JSON records with a schema", () =>
    Effect.gen(function* () {
      const consumed = fromConsumedRecords([
        { timestampMs: 1000, content: toBase64(new TextEncoder().encode('{"id":7}')) },
      ]);
      expect(consumed[0]!.timestamp.getTime()).toEqual(1000);
      expect(consumed[0]!.headers).toEqual({});
      const [decoded] = yield* decodeWithSchema(Order)(consumed);
      expect(decoded!.value).toEqual({ id: 7 });

      const bad = fromConsumedRecords([
        { timestampMs: 0, content: toBase64(new TextEncoder().encode("not json")) },
      ]);
      const error = yield* decodeWithSchema(Order)(bad).pipe(Effect.flip);
      expect(error._tag).toEqual("K2SchemaError");
    }),
  );

  it("packs requests under 5 MB of wire size, preserving order", () => {
    const records = Array.from({ length: 12 }, () => record(1_000_000));
    const batches = packStreamSinkBatches(records);
    expect(batches.map((b) => b.length)).toEqual([3, 3, 3, 3]);
    for (const batch of batches) {
      expect(batch.reduce((sum, r) => sum + wireSize(r), 0)).toBeLessThanOrEqual(MAX_REQUEST_BYTES);
    }
    expect(batches.flat()).toEqual(records);
  });

  it("ships an oversized record alone", () => {
    const huge = record(MAX_REQUEST_BYTES);
    expect(packStreamSinkBatches([record(1), huge, record(1)]).map((b) => b.length)).toEqual([
      1, 1, 1,
    ]);
  });

  it.effect("sends one request per chunk with JSON headers", () =>
    Effect.gen(function* () {
      const { client, sent } = fakeClient();
      yield* run(
        Stream.fromIterable([1, 2, 3, 4, 5].map((id) => ({ id }))).pipe(Stream.rechunk(2)),
        client,
      );
      expect(sent.map((b) => b.length)).toEqual([2, 2, 1]);
      expect(sent[0]![0]!.headers).toEqual({ "content-type": "application/json" });
      expect(new TextDecoder().decode(sent[2]![0]!.content)).toEqual('{"id":5}');
    }),
  );

  // `it.live`: the retry schedule sleeps on the real clock.
  it.live("retries K2Unavailable (not stored) a bounded number of times", () =>
    Effect.gen(function* () {
      const { client, sent, attempts } = fakeClient(["K2Unavailable", "K2Unavailable"]);
      yield* run(Stream.make({ id: 1 }), client);
      expect(attempts()).toEqual(3);
      expect(sent.length).toEqual(1);
    }),
  );

  it.effect("never retries K2AppendOutcomeUnknown (may have been stored)", () =>
    Effect.gen(function* () {
      const { client, attempts } = fakeClient(["K2AppendOutcomeUnknown"]);
      const error = yield* run(Stream.make({ id: 1 }), client).pipe(Effect.flip);
      expect(error._tag).toEqual("K2AppendOutcomeUnknown");
      expect(attempts()).toEqual(1);
    }),
  );

  it.effect("maps native binding results onto the HTTP API's typed tags", () =>
    Effect.gen(function* () {
      const binding = (
        result: Awaited<ReturnType<K2StreamBinding["send"]>> | Error,
      ): K2StreamBinding => ({
        send: () => (result instanceof Error ? Promise.reject(result) : Promise.resolve(result)),
      });
      const records = [{ content: new Uint8Array([1]) }];
      yield* sendViaBinding(binding({ success: true }), records);

      const failure = (code: number) =>
        sendViaBinding(
          binding({ success: false, error: { code, message: "nope", retryable: false } }),
          records,
        ).pipe(
          Effect.flip,
          Effect.map((e) => e._tag),
        );
      expect(yield* failure(10200)).toEqual("K2StreamNotFound");
      expect(yield* failure(10204)).toEqual("K2InvalidRequest");
      expect(yield* failure(10206)).toEqual("K2RequestTooLarge");
      expect(yield* failure(10207)).toEqual("K2RecordTooLarge");
      expect(yield* failure(10211)).toEqual("K2Unavailable");
      expect(yield* failure(10212)).toEqual("K2AppendOutcomeUnknown");
      expect(yield* failure(99999)).toEqual("UnknownCloudflareError");

      // A rejected RPC means the append outcome is unknown.
      const rejected = yield* sendViaBinding(binding(new Error("rpc reset")), records).pipe(
        Effect.flip,
      );
      expect(rejected._tag).toEqual("K2AppendOutcomeUnknown");
    }),
  );
});