sailfork / t3code-github / .repos/ alchemy-effect/ packages/ git/ src/ Protocol/ Sideband.ts

/// <reference types="@cloudflare/workers-types" />
/**
 * side-band-64k multiplexing.
 *
 * When `side-band-64k` is negotiated, binary data is carried as pkt-lines
 * whose first payload byte is the band:
 *
 * - **band 1** — pack data (or the receive-pack report)
 * - **band 2** — progress messages (client prints to stderr as `remote: `)
 * - **band 3** — fatal error message; the client aborts
 *
 * Frame arithmetic: max pkt-line total 65520 = 4 length bytes + 1 band byte
 * + **65515** data bytes, so every frame carries at most 65515 payload bytes.
 * Legacy `side-band` (1000-byte packets) is never advertised.
 */
import * as Stream from "effect/Stream";
import { concatBytes, utf8Encode } from "./ObjectCodec.ts";
import { flushPkt } from "./Pkt.ts";

/**
 * Maximum data bytes per side-band-64k frame (65520 − 4 length − 1 band).
 */
export const SIDEBAND_DATA_MAX = 65515;

/**
 * The three side-band channels.
 */
export type SidebandBand = 1 | 2 | 3;

/**
 * Encodes a single side-band frame. The data must be at most
 * {@link SIDEBAND_DATA_MAX} bytes — use {@link sidebandFrames} to split
 * arbitrary payloads.
 */
const HEX = "0123456789abcdef";

/** The 5-byte sideband frame header for `dataLength` payload bytes. */
const sidebandHeader = (band: SidebandBand, dataLength: number): Uint8Array => {
  const total = dataLength + 5;
  const out = new Uint8Array(5);
  out[0] = HEX.charCodeAt((total >> 12) & 0xf);
  out[1] = HEX.charCodeAt((total >> 8) & 0xf);
  out[2] = HEX.charCodeAt((total >> 4) & 0xf);
  out[3] = HEX.charCodeAt(total & 0xf);
  out[4] = band;
  return out;
};

export const sidebandFrame = (band: SidebandBand, data: Uint8Array): Uint8Array => {
  // One allocation, one copy: header + band + payload in place.
  const out = new Uint8Array(5 + data.length);
  out.set(sidebandHeader(band, data.length), 0);
  out.set(data, 5);
  return out;
};

/**
 * Splits a payload into as many side-band frames as needed (each carrying at
 * most {@link SIDEBAND_DATA_MAX} data bytes). An empty payload produces no
 * frames.
 */
export const sidebandFrames = (band: SidebandBand, data: Uint8Array): Array<Uint8Array> => {
  const frames: Array<Uint8Array> = [];
  for (let pos = 0; pos < data.length; pos += SIDEBAND_DATA_MAX) {
    frames.push(sidebandFrame(band, data.subarray(pos, pos + SIDEBAND_DATA_MAX)));
  }
  return frames;
};

/**
 * Stream transform: wraps every chunk of a byte stream into side-band frames
 * on the given band. Used to mux the pack stream (band 1) into the response
 * body; the caller appends the outer flush-pkt after the wrapped stream.
 */
export const wrapSideband =
  (band: SidebandBand) =>
  <E, R>(stream: Stream.Stream<Uint8Array, E, R>): Stream.Stream<Uint8Array, E, R> =>
    Stream.flatMap(stream, (chunk) =>
      chunk.length === 0 ? Stream.empty : Stream.fromArray(sidebandFrames(band, chunk)),
    );

/**
 * Encodes a band-2 progress message (e.g. `Counting objects: 42, done.`).
 * A trailing newline is appended when missing so the client's stderr stays
 * line-buffered.
 */
export const progressMessage = (message: string): Uint8Array =>
  concatBytes(
    sidebandFrames(
      2,
      utf8Encode(message.endsWith("\n") || message.endsWith("\r") ? message : `${message}\n`),
    ),
  );

/**
 * Encodes a band-3 fatal error message. The client prints it and aborts the
 * transfer; the server should end the response after sending it.
 */
export const sidebandError = (message: string): Uint8Array =>
  concatBytes(sidebandFrames(3, utf8Encode(message.endsWith("\n") ? message : `${message}\n`)));

/** Target bytes per native write from {@link pumpPackBody}. */
export const PUMP_READ_BYTES = 1 << 20;

/**
 * Sideband-frames a whole chunk into ONE buffer: `ceil(n / 65515)` frames,
 * each `header(5) + data`, laid out back to back. One memcpy per byte.
 */
export const sidebandFrameAll = (band: SidebandBand, data: Uint8Array) => {
  const frames = Math.ceil(data.length / SIDEBAND_DATA_MAX);
  const out = new Uint8Array(data.length + frames * 5);
  let at = 0;
  for (let pos = 0; pos < data.length; pos += SIDEBAND_DATA_MAX) {
    const end = Math.min(pos + SIDEBAND_DATA_MAX, data.length);
    out.set(sidebandHeader(band, end - pos), at);
    out.set(data.subarray(pos, end), at + 5);
    at += 5 + (end - pos);
  }
  return out;
};

/**
 * Pumps a pack body to the client through a native `IdentityTransformStream`
 * (DESIGN §22.3). Returns the response body.
 *
 * Why this shape, measured on a 40 MiB bundle (DESIGN §22.4):
 *
 * - An Effect `Stream` body tops out around 15–20 MiB/s: every chunk is a
 *   fiber round trip. The raw case is a platform-to-platform `pipeTo` —
 *   zero JS per chunk — and runs at the client's line rate (71 MiB/s here).
 * - Sideband cannot be a bare pipe: the protocol caps a frame at 65515
 *   data bytes. Writing frames one at a time into the identity stream —
 *   whether from a `TransformStream` or a manual loop — also measured
 *   ~15 MiB/s: the identity stream is unbuffered, so each `write` waits
 *   for the consumer, and 64 KiB per round trip is the ceiling.
 * - So: read ~1 MiB at a time (BYOB `readAtLeast` on byte sources, which
 *   R2 bodies are), frame the whole read into one buffer, one `write`.
 *   16× fewer round trips; one extra memcpy per byte (~1 ms/MiB).
 *
 * Errors on the source abort the writable, which errors the body the
 * client is reading — the right signal for a truncated pack.
 */
export const pumpPackBody = (options: {
  /** Bytes before the pack (NAK/ACK lines, progress) — sent as-is. */
  readonly prefix: Uint8Array;
  readonly source: ReadableStream<Uint8Array>;
  readonly sideband: boolean;
  /**
   * The source is ALREADY sideband-framed (a `bundleSidebandKey` object):
   * pipe it through natively and only append the flush packet.
   */
  readonly framed?: boolean | undefined;
}): ReadableStream<Uint8Array> => {
  const { readable, writable } = new IdentityTransformStream();
  void (async () => {
    const writer = writable.getWriter();
    try {
      await writer.write(options.prefix);
      if (!options.sideband) {
        writer.releaseLock();
        await options.source.pipeTo(writable);
        return;
      }
      if (options.framed === true) {
        writer.releaseLock();
        await options.source.pipeTo(writable, { preventClose: true });
        const tail = writable.getWriter();
        await tail.write(flushPkt);
        await tail.close();
        return;
      }
      for await (const chunk of readBig(options.source)) {
        await writer.write(sidebandFrameAll(1, chunk));
      }
      await writer.write(flushPkt);
      await writer.close();
    } catch (error) {
      await writable.abort(error).catch(() => {});
    }
  })();
  return readable;
};

/**
 * Yields chunks of about {@link PUMP_READ_BYTES} from a source: BYOB
 * `readAtLeast` (a workerd extension) fills a whole buffer per read on byte
 * streams; other streams are read as they come.
 */
async function* readBig(source: ReadableStream<Uint8Array>): AsyncGenerator<Uint8Array> {
  let byob: ReadableStreamBYOBReader | undefined;
  try {
    byob = source.getReader({ mode: "byob" });
  } catch {
    byob = undefined;
  }
  if (byob === undefined) {
    const reader = source.getReader();
    for (;;) {
      const { done, value } = await reader.read();
      if (done) return;
      if (value.length > 0) yield value;
    }
  }
  const reader = byob as ReadableStreamBYOBReader & {
    readAtLeast?: (min: number, view: Uint8Array) => Promise<ReadableStreamReadResult<Uint8Array>>;
  };
  let buffer = new ArrayBuffer(PUMP_READ_BYTES);
  for (;;) {
    const view = new Uint8Array(buffer);
    const result =
      reader.readAtLeast !== undefined
        ? await reader.readAtLeast(view.byteLength, view)
        : await reader.read(view);
    if (result.value !== undefined && result.value.byteLength > 0) {
      yield result.value;
      // The buffer was transferred into `value`; take it back for reuse.
      buffer = result.value.buffer as ArrayBuffer;
    }
    if (result.done) return;
  }
}

/** Byte length of {@link sidebandRechunk}'s output for `size` input bytes. */
export const sidebandFramedLength = (size: number): number =>
  size + 5 * Math.ceil(size / SIDEBAND_DATA_MAX);

/**
 * Re-frames a byte stream into band frames of EXACTLY
 * {@link SIDEBAND_DATA_MAX} data bytes (the last one shorter), independent
 * of how the input is chunked — so the output length is a pure function
 * of the input length ({@link sidebandFramedLength}), which an R2 put of
 * a stream needs up front. Used to write a bundle's pre-framed twin.
 */
export const sidebandRechunk =
  (band: SidebandBand) =>
  <E, R>(stream: Stream.Stream<Uint8Array, E, R>): Stream.Stream<Uint8Array, E, R> => {
    let carry: Uint8Array = new Uint8Array(0);
    const frames = (chunk: Uint8Array): Array<Uint8Array> => {
      const data = carry.length === 0 ? chunk : concatBytes([carry, chunk]);
      const out: Array<Uint8Array> = [];
      let pos = 0;
      while (data.length - pos >= SIDEBAND_DATA_MAX) {
        out.push(sidebandFrame(band, data.subarray(pos, pos + SIDEBAND_DATA_MAX)));
        pos += SIDEBAND_DATA_MAX;
      }
      carry = data.slice(pos);
      return out;
    };
    return Stream.flatMap(stream, (chunk) => Stream.fromArray(frames(chunk))).pipe(
      Stream.concat(
        Stream.suspend(() =>
          carry.length === 0 ? Stream.empty : Stream.succeed(sidebandFrame(band, carry)),
        ),
      ),
    );
  };