sailfork / t3code-github / .repos/ alchemy-effect/ packages/ git/ src/ Protocol/ PackWriter.ts

/**
 * No-delta pack emission (DESIGN §3.5) — streaming, constant memory:
 *
 * ```
 * PACK + version(2) + count(manifest.length)     — count known upfront
 * for each entry:  varint(type, size) + zdata    — stored zlib bytes, verbatim
 * sha1 trailer                                   — incremental over everything
 * ```
 *
 * No compression CPU at all: objects are stored pre-deflated (`zdata` is
 * `zlib(content)` without the loose header), so pack generation is literally
 * concatenation + SHA-1. Serving no deltas / no thin packs is legal
 * (thin-pack/ofs-delta are protocol MAYs).
 *
 * The writer emits {@link PackEvent}s — raw pack bytes plus optional progress
 * marks — so the choreography layer can mux data onto side-band 1 and
 * progress onto band 2 without this module knowing about sideband framing.
 */
import * as Effect from "effect/Effect";
import * as Stream from "effect/Stream";
import { encodeTypeSize, makeSha1, utf8Encode } from "./ObjectCodec.ts";
import type { ManifestEntry, ObjectSource, StoreError } from "./Store.ts";

/**
 * One item of the pack emission stream: raw pack bytes, or a progress mark
 * after an entry finished (`written` of `total` objects emitted).
 */
export type PackEvent =
  | { readonly _tag: "data"; readonly bytes: Uint8Array }
  | {
      readonly _tag: "progress";
      readonly written: number;
      readonly total: number;
    };

/**
 * Builds the 12-byte pack header: `"PACK"`, version 2, entry count (both
 * 32-bit big-endian).
 */
export const packHeader = (count: number): Uint8Array => {
  const out = new Uint8Array(12);
  out.set(utf8Encode("PACK"), 0);
  const view = new DataView(out.buffer);
  view.setUint32(4, 2);
  view.setUint32(8, count);
  return out;
};

/**
 * Options for {@link writePack}.
 */
export interface WritePackOptions {
  /**
   * Emit a `progress` event after every N entries (and after the last). When
   * omitted, no progress events are emitted.
   */
  readonly progressEvery?: number | undefined;
}

/**
 * Streams a complete no-delta pack for the given manifest: header + count,
 * per-object `varint(type, size)` + stored zlib bytes (read from the
 * {@link ObjectSource}), and the incremental SHA-1 trailer.
 *
 * The manifest is materialized by the closure walk before streaming so the
 * header count is exact. Emission is sequential and constant-memory: one
 * object's zdata chunks are in flight at a time.
 */
export const writePack = (
  entries: ReadonlyArray<ManifestEntry>,
  objects: ObjectSource,
  options?: WritePackOptions,
): Stream.Stream<PackEvent, StoreError> =>
  Stream.unwrap(
    Effect.sync(() => {
      const sha = makeSha1();
      const total = entries.length;
      const progressEvery = options?.progressEvery;
      let written = 0;

      const data = (bytes: Uint8Array): PackEvent => {
        sha.update(bytes);
        return { _tag: "data", bytes };
      };

      const header: Stream.Stream<PackEvent> = Stream.fromEffect(
        Effect.sync(() => data(packHeader(total))),
      );

      const body: Stream.Stream<PackEvent, StoreError> = Stream.flatMap(
        Stream.fromArray(entries),
        (entry) => {
          const head = Stream.fromEffect(
            Effect.sync(() => data(encodeTypeSize(entry.type, entry.size))),
          );
          const zdata = Stream.map(objects.readZData(entry.oid), data);
          const progress: Stream.Stream<PackEvent> = Stream.unwrap(
            Effect.sync(() => {
              written += 1;
              return progressEvery !== undefined &&
                (written % progressEvery === 0 || written === total)
                ? Stream.succeed<PackEvent>({
                    _tag: "progress",
                    written,
                    total,
                  })
                : Stream.empty;
            }),
          );
          return Stream.concat(Stream.concat(head, zdata), progress);
        },
      );

      const trailer: Stream.Stream<PackEvent> = Stream.fromEffect(
        Effect.sync((): PackEvent => ({ _tag: "data", bytes: sha.digest() })),
      );

      return Stream.concat(Stream.concat(header, body), trailer);
    }),
  );

/**
 * Drops progress events, keeping only the raw pack bytes (the non-sideband
 * response shape).
 */
export const packDataBytes = <E, R>(
  events: Stream.Stream<PackEvent, E, R>,
): Stream.Stream<Uint8Array, E, R> =>
  Stream.flatMap(events, (event) =>
    event._tag === "data" ? Stream.succeed(event.bytes) : Stream.empty,
  );

/**
 * Convenience: the raw pack byte stream for a manifest, with no progress
 * events.
 */
export const writePackBytes = (
  entries: ReadonlyArray<ManifestEntry>,
  objects: ObjectSource,
): Stream.Stream<Uint8Array, StoreError> => packDataBytes(writePack(entries, objects));