sailfork / t3code-github / .repos/ alchemy-effect/ packages/ git/ src/ Store/ ObjectStore.ts

import { RuntimeContext } from "alchemy/RuntimeContext";
import * as Effect from "effect/Effect";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";
/**
 * The concrete object store: DO SQLite rows + R2 overflow (DESIGN.md §3.3).
 *
 * Implements the git layer's `ObjectSource` contract over the `objects`
 * table, dispatching byte reads on the `location` column:
 *
 * - `'row'`  — the `zdata` BLOB (zlib(content), no loose header),
 * - `'r2'`   — a streamed R2 `get` of `r2_key` (which may point into a fork
 *   parent's prefix — immutable, shared),
 * - `'pack'` — v1.1 compaction; reads die with {@link NotImplementedError}
 *   until the compaction alarm lands (schema-ready, DESIGN.md §10).
 *
 * Plus the push-side surface the protocol layer needs: staged inserts with
 * the row/R2 placement policy, batched metadata reads, and the
 * staged-∪-live membership oracle behind the receive-pack connectivity
 * check (DESIGN.md §3.6 step 5).
 *
 * All reads on the `ObjectSource` surface see **live objects only**
 * (`staged_push IS NULL`); staged rows become visible to fetches only after
 * the final `transactionSync` flips them live.
 */
import type { BlobStoreError, BlobStoreShape } from "../BlobStore.ts";
import type { ObjectType, Oid } from "../Protocol/ObjectCodec.ts";
import { concatBytes, encodeTypeSize, type PackEntryType } from "../Protocol/ObjectCodec.ts";
import type { ManifestEntry } from "../Protocol/Store.ts";
import {
  StoreError,
  type ObjectLocation,
  type ObjectMeta,
  type ObjectSource,
} from "../Protocol/Store.ts";
import * as Zlib from "../Protocol/Zlib.ts";
import { objectKey, packKeyOf } from "./Keys.ts";
import type { ObjectMetaRow, SqlClient } from "./Sql.ts";

// ─────────────────────────────────────────────────────────────────────────────
// Constants & errors
// ─────────────────────────────────────────────────────────────────────────────

/**
 * Compressed-size threshold above which an object's bytes are offloaded to
 * R2 (`location = 'r2'`) instead of a row BLOB — 1 MiB, a safety margin
 * under the 2 MB SQLite row cap (DESIGN.md §3.3).
 */
export const R2_OFFLOAD_THRESHOLD = 1_048_576;

/**
 * Per-object uncompressed size cap for v1 — 64 MiB. Delta application needs
 * base + result in memory; this keeps worst-case ingest memory bounded.
 * Enforced by `PackParser` (rejected with `unpack object too large`);
 * exported here as the storage layer's authoritative constant.
 */
export const MAX_OBJECT_SIZE = 67_108_864;

/**
 * Window size for coalesced reads out of compacted packs (DESIGN §15
 * bottleneck 2). Touching one object in a window fetches the whole window,
 * so a fetch that walks thousands of objects costs a handful of R2 GETs
 * instead of thousands.
 */
export const WINDOW_BYTES = 4 * 1024 * 1024;

/**
 * Objects per batch on the emission/closure paths — one SQL statement and
 * one emitted chunk per batch instead of per object (DESIGN §22).
 */
export const EMIT_BATCH = 256;

/**
 * Rows per multi-row `INSERT` while staging a push: 6 bindings per row, and
 * Durable Object SQLite allows at most 100 bound parameters per statement
 * (`MAX_IN_PARAMS`) — 16 rows is 96. Still a 16x cut in statement
 * executions over one row per statement (see `insertStagedBatch`).
 */
export const STAGE_INSERT_ROWS = 16;

const chunkArray = <T>(items: ReadonlyArray<T>, size: number): Array<Array<T>> => {
  const out: Array<Array<T>> = [];
  for (let i = 0; i < items.length; i += size) {
    out.push(items.slice(i, i + size) as Array<T>);
  }
  return out;
};

/**
 * Isolate-wide budget for retained pack windows — 32 MiB (8 windows).
 *
 * The cache is a module-level LRU shared by every repo instance in the
 * isolate, not a per-instance map: Durable Object instances of one class
 * share an isolate and its 128 MiB, so per-instance caches multiply with
 * the number of active repos. Measured (DESIGN §22.4): a 36 MiB push into a
 * second repo was rejected with "isolate exceeded its memory limit" while
 * the first repo's caches were resident. Keys carry the repo id
 * (`packKey`), so sharing is safe.
 */
export const WINDOW_CACHE_BYTES = 32 * 1024 * 1024;

/**
 * Windows fetched ahead of the one being emitted (DESIGN §22.4). Blob
 * storage GETs cost ~150–250 ms each regardless of size, and a 40 MiB
 * pack is ten of them; fetched one at a time they were half of the
 * dynamic transfer time. 3 ahead + 1 current stays within the cache.
 */
export const PACK_LOOKAHEAD = 3;

const windows = new Map<string, { start: number; bytes: Uint8Array }>();
let windowBytes = 0;
const retainWindow = (cacheKey: string, slab: { start: number; bytes: Uint8Array }) => {
  windows.set(cacheKey, slab);
  windowBytes += slab.bytes.byteLength;
  while (windowBytes > WINDOW_CACHE_BYTES && windows.size > 1) {
    const oldest = windows.keys().next().value;
    if (oldest === undefined) break;
    windowBytes -= windows.get(oldest)!.bytes.byteLength;
    windows.delete(oldest);
  }
};

/**
 * Raised (as a defect, via `Effect.die`) when a v1 code path reaches a
 * feature the design explicitly defers — currently only reads of
 * `location = 'pack'` objects, which exist starting with v1.1 compaction.
 */
export class NotImplementedError extends Schema.TaggedError<NotImplementedError>()(
  "NotImplementedError",
  { feature: Schema.String },
) {}

// ─────────────────────────────────────────────────────────────────────────────
// Contract
// ─────────────────────────────────────────────────────────────────────────────

/**
 * A pack entry resolved by `PackParser`, ready to stage: the object id, its
 * numeric type, uncompressed size, and the zlib-compressed bytes to store
 * (the pack's compressed span verbatim for non-delta entries; a fresh
 * `deflate(content)` for delta-resolved ones).
 */
/**
 * The live-object predicate (DESIGN §22.14). A push's rows are live the
 * moment its `pushes` row is committed; the per-row `staged_push` column
 * is nulled afterwards, off the response path (`flipPush`), and by GC as a
 * safety net. Reads never wait for that rewrite.
 */
export const LIVE_OBJECTS =
  "(staged_push IS NULL OR staged_push IN (SELECT push_id FROM pushes WHERE state = 'committed'))";

export interface StagedObject {
  readonly oid: Oid;
  /**
   * Set when the bytes already live in a pack in blob storage (a promoted
   * wire pack, DESIGN §22.5): the row is staged as `location='pack'` with
   * these coordinates and `zdata` is NOT stored.
   */
  readonly pack?: { readonly packId: string; readonly offset: number } | undefined;
  /**
   * Compressed span when `zdata` is not carried (a promoted row whose
   * bytes live in the pack): the insert must record the real size.
   */
  readonly zsize?: number | undefined;
  readonly type: ObjectType;
  /** Uncompressed content size in bytes. */
  readonly size: number;
  /** `zlib(content)` without the loose header. */
  readonly zdata: Uint8Array;
}

/** Outcome of {@link ObjectStore.insertStaged}. */
export type InsertOutcome = "inserted" | "exists";

/**
 * The full storage surface of a repo's objects: the read-only
 * `ObjectSource` the protocol layer consumes, plus batched metadata reads,
 * staged inserts, and the connectivity-check membership oracle.
 */
export interface ObjectStore extends ObjectSource {
  /**
   * Inflated content for many objects, one statement per ~256 oids — the
   * closure walk reads whole tree levels this way instead of one SQL
   * round trip per tree (DESIGN §22). Missing oids are simply absent.
   */
  readonly readContentBatch: (
    oids: ReadonlyArray<Oid>,
  ) => Effect.Effect<ReadonlyMap<Oid, Uint8Array>, StoreError>;
  readonly packEntries: (
    entries: ReadonlyArray<ManifestEntry>,
  ) => Stream.Stream<Uint8Array, StoreError>;
  /**
   * Batched metadata for many oids (live objects only). Chunked `IN`
   * queries, ≤ 100 params each. Missing oids are simply absent from the map.
   */
  readonly getMetaBatch: (
    oids: ReadonlyArray<Oid>,
  ) => Effect.Effect<ReadonlyMap<Oid, ObjectMeta>, StoreError>;
  /**
   * Stages one resolved object for `pushId` (DESIGN.md §3.6 step 4).
   *
   * Placement policy: compressed size > 1 MiB ⇒ the bytes are written to R2
   * **first** (`{repoId}/objects/{oid}`, content-addressed so re-runs are
   * idempotent), then the metadata row; otherwise the row carries the BLOB.
   *
   * Existing oids are skipped (idempotent re-push). A row left staged by a
   * **different** (crashed) push is adopted — re-stamped with this `pushId`
   * so the connectivity check and the final live-flip see it.
   */
  readonly insertStaged: (
    pushId: string,
    object: StagedObject,
  ) => Effect.Effect<InsertOutcome, StoreError>;
  /**
   * Batched sibling of {@link ObjectStore.insertStaged} — the ingest hot
   * path (DESIGN.md §16.6).
   *
   * Staging objects one at a time costs two statements each (an existence
   * probe and an insert), so a 13.7k-object push ran ~27k statements and
   * ~13.7k transactions inside a single-threaded Durable Object. This
   * inserts a whole batch in **one** `transactionSync` with
   * `INSERT OR IGNORE` (no probe), then adopts any rows left staged by a
   * crashed push with one chunked `UPDATE`.
   *
   * Oversize objects (> 1 MiB compressed) still go through the per-object
   * path, since each needs its own R2 write first.
   */
  readonly insertStagedBatch: (
    pushId: string,
    objects: ReadonlyArray<StagedObject>,
  ) => Effect.Effect<void, StoreError>;
  /** A live object's type and content for thin-delta resolution, or `undefined`. */
  /** Read only live objects or objects staged by this push, with a caller-selected size cap. */
  readonly readPrepared: (
    pushId: string,
    oid: Oid,
    maxBytes: number,
  ) => Effect.Effect<
    { readonly type: ObjectType; readonly content: Uint8Array } | undefined,
    StoreError
  >;
  readonly readBase: (
    oid: Oid,
  ) => Effect.Effect<
    { readonly type: ObjectType; readonly content: Uint8Array } | undefined,
    StoreError
  >;
  /**
   * The connectivity-check membership oracle: returns the subset of `oids`
   * that exist in **neither** the live store **nor** this push's staged
   * rows. An empty result means every referenced object is present and the
   * push is safe to commit.
   */
  readonly missingObjects: (
    oids: ReadonlyArray<Oid>,
    pushId: string,
  ) => Effect.Effect<ReadonlyArray<Oid>, StoreError>;
}

/**
 * Dependencies of {@link makeObjectStore}. Plain values (not Layers) so
 * `RepoObject.ts` wires the store inside its inner per-instance effect.
 */
export interface ObjectStoreOptions {
  /** The repo DO's SQL client (see `makeSqlClient`). */
  readonly sql: SqlClient;
  /** The swappable bulk-byte store (R2 by default — see `BlobStore.ts`). */
  readonly blobs: BlobStoreShape;
  /** The repo's ULID — the R2 key prefix (DESIGN.md §3.2). */
  readonly repoId: string;
}

// ─────────────────────────────────────────────────────────────────────────────
// Implementation
// ─────────────────────────────────────────────────────────────────────────────

const VALID_LOCATIONS: ReadonlySet<string> = new Set(["row", "r2", "pack"]);

const isObjectType = (type: number): type is ObjectType =>
  type === 1 || type === 2 || type === 3 || type === 4;

const metaFromRow = (row: ObjectMetaRow): Effect.Effect<ObjectMeta, StoreError> => {
  if (!isObjectType(row.type) || !VALID_LOCATIONS.has(row.location)) {
    return Effect.fail(
      StoreError.make({
        reason: `corrupt objects row for ${row.oid}: type=${row.type} location=${row.location}`,
      }),
    );
  }
  return Effect.succeed({
    oid: row.oid,
    type: row.type,
    size: row.size,
    zsize: row.zsize,
    location: row.location as ObjectLocation,
  });
};

/**
 * Returns a plain `ArrayBuffer` view of the bytes for SQLite BLOB binding
 * (workerd binds `ArrayBuffer`, not typed arrays). Zero-copy when the view
 * spans its whole buffer.
 */
const toArrayBuffer = (bytes: Uint8Array): ArrayBuffer =>
  bytes.byteOffset === 0 &&
  bytes.byteLength === bytes.buffer.byteLength &&
  bytes.buffer instanceof ArrayBuffer
    ? bytes.buffer
    : (bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer);

/** Row projection used by the byte-read dispatch. */
interface ZDataRow extends Record<string, string | number | ArrayBuffer | null> {
  readonly oid: string;
  readonly location: string;
  readonly zdata: ArrayBuffer | null;
  readonly r2_key: string | null;
  /** sha1 of the containing pack when `location='pack'` (DESIGN §12.1). */
  readonly pack_id: string | null;
  /** Byte offset of this object's zdata span inside that pack. */
  readonly pack_offset: number | null;
  /** Length of the zdata span (mirrors `zsize`). */
  readonly zsize: number;
}

/**
 * Creates the {@link ObjectStore} for one repo. Plain factory — no Layers —
 * so `RepoObject.ts` can wire it in its inner effect with the ambient
 * `state`/bucket handles.
 */
export const makeObjectStore = (options: ObjectStoreOptions): ObjectStore => {
  const { sql, blobs, repoId } = options;

  /** Discharges the blob store's `RuntimeContext` coloring and closes the
   * error union into `StoreError` (we run inside the Repo DO). */
  const runBlob = <A>(
    effect: Effect.Effect<A, BlobStoreError, RuntimeContext>,
    what: string,
  ): Effect.Effect<A, StoreError> =>
    effect.pipe(
      Effect.mapError((error) => StoreError.make({ reason: `${what}: ${error.reason}` })),
      Effect.provide(RuntimeContext.phantom),
    );

  const zlibToStore = (error: Zlib.ZlibError): StoreError =>
    StoreError.make({ reason: error.reason });

  const readRow = (
    oid: Oid,
    pushId: string | null = null,
  ): Effect.Effect<ZDataRow | undefined, StoreError> =>
    sql.first<ZDataRow>(
      `SELECT oid, location, zdata, r2_key, pack_id, pack_offset, zsize
         FROM objects WHERE oid = ? AND (${LIVE_OBJECTS} OR staged_push = ?)`,
      oid,
      pushId,
    );

  const requireRow = Effect.fn(function* (oid: Oid, pushId: string | null = null) {
    const row = yield* readRow(oid, pushId);
    if (row === undefined) {
      return yield* StoreError.make({ reason: `object not found: ${oid}` });
    }
    return row;
  });

  /** Fetches the R2 body for an `'r2'`-located row. */
  const r2Body = Effect.fn(function* (oid: Oid, r2Key: string | null) {
    if (r2Key === null) {
      return yield* StoreError.make({
        reason: `object ${oid} has location='r2' but no r2_key`,
      });
    }
    const body = yield* runBlob(blobs.get(r2Key), `blob get ${r2Key}`);
    if (body === null) {
      return yield* StoreError.make({ reason: `R2 object missing: ${r2Key}` });
    }
    return body;
  });

  /**
   * Coalesced window cache over compacted packs (DESIGN §15 bottleneck 2).
   *
   * A pack read is one ranged R2 GET per object if done naively — and a
   * fetch that walks a few thousand objects then costs a few thousand
   * serialized round trips. Measured: **68 s to serve a 53 KiB fetch** over
   * a 1202-object pack. Instead, reads are served out of window-aligned
   * slabs: touching any object in a window fetches the whole window once,
   * and every other object inside it is then free. A small pack collapses
   * to a single GET.
   *
   * Bounded by construction: at most {@link MAX_CACHED_WINDOWS} windows of
   * {@link WINDOW_BYTES} are retained (insertion-ordered LRU), so the cache
   * cannot outgrow the isolate's memory budget.
   */

  const readWindow = Effect.fn(function* (key: string, windowIndex: number) {
    const cacheKey = `${key}:${windowIndex}`;
    const hit = windows.get(cacheKey);
    if (hit !== undefined) {
      // Refresh recency (Map iterates in insertion order).
      windows.delete(cacheKey);
      windows.set(cacheKey, hit);
      return hit;
    }
    const start = windowIndex * WINDOW_BYTES;
    const body = yield* runBlob(
      blobs.get(key, { offset: start, length: WINDOW_BYTES }),
      `blob window get ${key}`,
    );
    if (body === null) {
      return yield* StoreError.make({ reason: `pack missing: ${key}` });
    }
    const bytes = yield* body.bytes.pipe(
      Effect.mapError((error) => StoreError.make({ reason: `R2 read ${key}: ${error.message}` })),
    );
    const slab = { start, bytes };
    retainWindow(cacheKey, slab);
    return slab;
  });

  /**
   * Reads an object's zdata span out of a compacted R2 pack (DESIGN §12.1),
   * via the window cache above.
   *
   * `pack_offset` addresses the object's **zdata** directly (not the pack
   * entry header), so the slice is exactly the stored compressed bytes —
   * the same bytes a `location='row'` object holds in its BLOB. Entry
   * headers are regenerated from `(type, size)` at emission time.
   */
  const packBytes = Effect.fn(function* (oid: Oid, row: ZDataRow) {
    if (row.pack_id === null || row.pack_offset === null) {
      return yield* StoreError.make({
        reason: `object ${oid} has location='pack' but no pack coordinates`,
      });
    }
    const key = packKeyOf(repoId, row.pack_id);
    const offset = row.pack_offset;
    const windowIndex = Math.floor(offset / WINDOW_BYTES);
    const slab = yield* readWindow(key, windowIndex);
    const relative = offset - slab.start;
    if (relative >= 0 && relative + row.zsize <= slab.bytes.length) {
      // Copy out: the slab is retained by the cache and must not be aliased.
      return slab.bytes.slice(relative, relative + row.zsize);
    }
    // The object straddles the window boundary (or the window was short) —
    // fall back to an exact ranged read for this one object.
    const body = yield* runBlob(
      blobs.get(key, { offset, length: row.zsize }),
      `blob ranged get ${key}`,
    );
    if (body === null) {
      return yield* StoreError.make({ reason: `pack missing: ${key}` });
    }
    return yield* body.bytes.pipe(
      Effect.mapError((error) => StoreError.make({ reason: `R2 read ${key}: ${error.message}` })),
    );
  });

  /** Reads the stored compressed bytes fully into memory. */
  const readZBytes = Effect.fn(function* (oid: Oid, pushId: string | null = null) {
    const row = yield* requireRow(oid, pushId);
    switch (row.location) {
      case "row": {
        if (row.zdata === null) {
          return yield* StoreError.make({
            reason: `object ${oid} has location='row' but NULL zdata`,
          });
        }
        return new Uint8Array(row.zdata);
      }
      case "r2": {
        const body = yield* r2Body(oid, row.r2_key);
        return yield* body.bytes.pipe(
          Effect.mapError((error) =>
            StoreError.make({
              reason: `R2 read ${row.r2_key}: ${error.message}`,
            }),
          ),
        );
      }
      case "pack":
        return yield* packBytes(oid, row);
      default:
        return yield* StoreError.make({
          reason: `object ${oid} has unknown location '${row.location}'`,
        });
    }
  });

  const getMetaBatch = (
    oids: ReadonlyArray<Oid>,
  ): Effect.Effect<ReadonlyMap<Oid, ObjectMeta>, StoreError> =>
    Effect.gen(function* () {
      const unique = Array.from(new Set(oids));
      const map = new Map<Oid, ObjectMeta>();
      if (unique.length === 0) {
        return map;
      }
      const rows = yield* sql.inChunks<ObjectMetaRow>(
        (ph) =>
          `SELECT oid, type, size, zsize, location FROM objects WHERE oid IN (${ph}) AND ${LIVE_OBJECTS}`,
        unique,
      );
      for (const row of rows) {
        map.set(row.oid, yield* metaFromRow(row));
      }
      return map;
    });

  const insertStagedOne = Effect.fn(function* (pushId: string, object: StagedObject) {
    const existing = yield* sql.first<{ staged_push: string | null }>(
      `SELECT staged_push FROM objects WHERE oid = ?`,
      object.oid,
    );
    if (existing !== undefined) {
      if (existing.staged_push !== null && existing.staged_push !== pushId) {
        // Adopt a row left staged by a crashed push so this push's
        // connectivity check and live-flip see it.
        yield* sql.run(
          `UPDATE objects SET staged_push = ? WHERE oid = ? AND staged_push IS NOT NULL`,
          pushId,
          object.oid,
        );
      }
      return "exists" as const;
    }
    if (object.zdata.byteLength > R2_OFFLOAD_THRESHOLD) {
      // R2 write FIRST, then the metadata row — a crash between the two
      // leaves only a harmless content-addressed R2 orphan.
      const key = objectKey(repoId, object.oid);
      yield* runBlob(blobs.put(key, object.zdata), `blob put ${key}`);
      yield* sql.run(
        `INSERT OR IGNORE INTO objects (oid, type, size, zsize, location, zdata, r2_key, staged_push)
           VALUES (?, ?, ?, ?, 'r2', NULL, ?, ?)`,
        object.oid,
        object.type,
        object.size,
        object.zdata.byteLength,
        key,
        pushId,
      );
    } else {
      yield* sql.run(
        `INSERT OR IGNORE INTO objects (oid, type, size, zsize, location, zdata, staged_push)
           VALUES (?, ?, ?, ?, 'row', ?, ?)`,
        object.oid,
        object.type,
        object.size,
        object.zdata.byteLength,
        toArrayBuffer(object.zdata),
        pushId,
      );
    }
    return "inserted" as const;
  });

  const readZDataOf = (oid: Oid): Stream.Stream<Uint8Array, StoreError> =>
    Stream.unwrap(
      Effect.gen(function* () {
        const row = yield* requireRow(oid);
        switch (row.location) {
          case "row": {
            if (row.zdata === null) {
              return yield* StoreError.make({
                reason: `object ${oid} has location='row' but NULL zdata`,
              });
            }
            return Stream.succeed(new Uint8Array(row.zdata));
          }
          case "r2": {
            const body = yield* r2Body(oid, row.r2_key);
            return body.stream.pipe(
              Stream.mapError((error) =>
                StoreError.make({
                  reason: `R2 stream ${row.r2_key}: ${error.message}`,
                }),
              ),
            );
          }
          case "pack":
            return Stream.succeed(yield* packBytes(oid, row));
          default:
            return yield* StoreError.make({
              reason: `object ${oid} has unknown location '${row.location}'`,
            });
        }
      }),
    );

  const self: ObjectStore = {
    // ── ObjectSource (live objects only) ────────────────────────────────────
    has: (oid) =>
      sql
        .first<{ one: number }>(
          `SELECT 1 AS one FROM objects WHERE oid = ? AND ${LIVE_OBJECTS}`,
          oid,
        )
        .pipe(Effect.map((row) => row !== undefined)),

    filterExisting: (oids) =>
      Effect.gen(function* () {
        const unique = Array.from(new Set(oids));
        if (unique.length === 0) {
          return [];
        }
        const rows = yield* sql.inChunks<{ oid: string }>(
          (ph) => `SELECT oid FROM objects WHERE oid IN (${ph}) AND ${LIVE_OBJECTS}`,
          unique,
        );
        return rows.map((row) => row.oid);
      }),

    getMeta: (oid) =>
      Effect.gen(function* () {
        const row = yield* sql.first<ObjectMetaRow>(
          `SELECT oid, type, size, zsize, location FROM objects WHERE oid = ? AND ${LIVE_OBJECTS}`,
          oid,
        );
        if (row === undefined) {
          return undefined;
        }
        return yield* metaFromRow(row);
      }),

    readZData: readZDataOf,

    readContent: (oid) =>
      readZBytes(oid).pipe(
        Effect.flatMap((zdata) => Zlib.inflate(zdata).pipe(Effect.mapError(zlibToStore))),
      ),

    readContentBatch: (oids) =>
      Effect.gen(function* () {
        const out = new Map<Oid, Uint8Array>();
        const unique = Array.from(new Set(oids));
        if (unique.length === 0) return out;
        const rows = yield* sql.inChunks<ZDataRow>(
          (ph) =>
            `SELECT oid, location, zdata, r2_key, pack_id, pack_offset, zsize
               FROM objects WHERE oid IN (${ph}) AND ${LIVE_OBJECTS}`,
          unique,
        );
        const inflate = (z: Uint8Array) => Zlib.inflate(z).pipe(Effect.mapError(zlibToStore));
        // Pack-resident rows in pack order so consecutive window reads hit.
        const packRows = rows
          .filter((row) => row.location === "pack")
          .sort(
            (a, b) =>
              (a.pack_id ?? "").localeCompare(b.pack_id ?? "") ||
              (a.pack_offset ?? 0) - (b.pack_offset ?? 0),
          );
        for (const row of rows) {
          if (row.location === "row" && row.zdata !== null) {
            out.set(row.oid, yield* inflate(new Uint8Array(row.zdata)));
          } else if (row.location === "r2") {
            out.set(row.oid, yield* readZBytes(row.oid).pipe(Effect.flatMap(inflate)));
          }
        }
        for (const row of packRows) {
          out.set(row.oid, yield* inflate(yield* packBytes(row.oid, row)));
        }
        return out;
      }),

    packEntries: (entries) =>
      Stream.suspend(() => {
        // Locality order (DESIGN §22): a non-delta pack is order-free, so
        // emit SQLite rows in manifest order, then every pack-resident
        // object sorted by (pack, offset) so the window cache streams each
        // 4 MiB window exactly once, then the rare R2-offloaded objects.
        const byOid = new Map(entries.map((entry) => [entry.oid, entry]));
        const rowEntries: Array<ManifestEntry> = [];
        const packList: Array<ManifestEntry> = [];
        const r2Entries: Array<ManifestEntry> = [];
        for (const entry of entries) {
          (entry.location === "row"
            ? rowEntries
            : entry.location === "pack"
              ? packList
              : r2Entries
          ).push(entry);
        }
        const header = (entry: ManifestEntry) =>
          encodeTypeSize(entry.type as PackEntryType, entry.size);

        const rowStream = Stream.fromIterable(chunkArray(rowEntries, EMIT_BATCH)).pipe(
          Stream.mapEffect((run) =>
            Effect.gen(function* () {
              const rows = yield* sql.inChunks<{
                oid: string;
                zdata: ArrayBuffer | null;
              }>(
                (ph) => `SELECT oid, zdata FROM objects WHERE oid IN (${ph}) AND ${LIVE_OBJECTS}`,
                run.map((entry) => entry.oid),
              );
              const zdataOf = new Map(rows.map((row) => [row.oid, row.zdata]));
              const pieces: Array<Uint8Array> = [];
              for (const entry of run) {
                const zdata = zdataOf.get(entry.oid);
                if (zdata === undefined || zdata === null) {
                  // The manifest said 'row', but compaction may have moved
                  // the bytes to a pack between closure and emission (the
                  // alarm runs concurrently with fetches). Re-dispatch on
                  // the CURRENT location; only a truly absent object fails.
                  pieces.push(header(entry), yield* readZBytes(entry.oid));
                  continue;
                }
                pieces.push(header(entry), new Uint8Array(zdata));
              }
              return concatBytes(pieces);
            }),
          ),
        );

        // Pack runs (DESIGN §22.4): coords sorted by (pack, offset) and
        // grouped by 4 MiB window. Each group is ONE window fetch (with the
        // next window fetched concurrently — a cache hit on the following
        // step) and one synchronous slice loop: no per-object Effect call,
        // no per-object cache lookup. Objects that straddle a window edge
        // (rare) fall back to the ranged read in `packBytes`; pack order
        // is free, so they simply follow their window's run.
        const packStream = Stream.unwrap(
          Effect.gen(function* () {
            if (packList.length === 0) return Stream.empty;
            const coords = yield* sql.inChunks<{
              oid: string;
              pack_id: string | null;
              pack_offset: number | null;
              zsize: number;
            }>(
              (ph) =>
                `SELECT oid, pack_id, pack_offset, zsize FROM objects WHERE oid IN (${ph}) AND ${LIVE_OBJECTS}`,
              packList.map((entry) => entry.oid),
            );
            coords.sort(
              (a, b) =>
                (a.pack_id ?? "").localeCompare(b.pack_id ?? "") ||
                (a.pack_offset ?? 0) - (b.pack_offset ?? 0),
            );
            type Coord = (typeof coords)[number];
            type Run = {
              readonly key: string;
              readonly window: number;
              readonly inside: Array<Coord>;
              readonly straddling: Array<Coord>;
            };
            const runs: Array<Run> = [];
            for (const coord of coords) {
              if (coord.pack_id === null || coord.pack_offset === null) {
                return yield* StoreError.make({
                  reason: `object ${coord.oid} is location='pack' without coordinates`,
                });
              }
              const key = packKeyOf(repoId, coord.pack_id);
              const window = Math.floor(coord.pack_offset / WINDOW_BYTES);
              let run = runs[runs.length - 1];
              if (run === undefined || run.key !== key || run.window !== window) {
                run = { key, window, inside: [], straddling: [] };
                runs.push(run);
              }
              (coord.pack_offset + coord.zsize <= (window + 1) * WINDOW_BYTES
                ? run.inside
                : run.straddling
              ).push(coord);
            }
            const rowOf = (coord: Coord): ZDataRow => ({
              oid: coord.oid,
              location: "pack",
              zdata: null,
              r2_key: null,
              pack_id: coord.pack_id,
              pack_offset: coord.pack_offset,
              zsize: coord.zsize,
            });
            // Lookahead: the next PACK_LOOKAHEAD windows are fetched
            // concurrently with this one; each is a cache hit when its
            // turn comes. Bounded by WINDOW_CACHE_BYTES (8 windows).
            return Stream.fromIterable(
              runs.map((run, i) => [run, runs.slice(i + 1, i + 1 + PACK_LOOKAHEAD)] as const),
            ).pipe(
              Stream.mapEffect(([run, ahead]) =>
                Effect.gen(function* () {
                  const [slab] = yield* Effect.all(
                    [
                      readWindow(run.key, run.window),
                      ...ahead.map((next) => readWindow(next.key, next.window).pipe(Effect.ignore)),
                    ],
                    { concurrency: 1 + PACK_LOOKAHEAD },
                  );
                  const pieces = yield* Effect.sync(() => {
                    const out: Array<Uint8Array> = [];
                    for (const coord of run.inside) {
                      const entry = byOid.get(coord.oid);
                      if (entry === undefined) continue;
                      const at = coord.pack_offset! - slab.start;
                      out.push(header(entry), slab.bytes.subarray(at, at + coord.zsize));
                    }
                    return out;
                  });
                  for (const coord of run.straddling) {
                    const entry = byOid.get(coord.oid);
                    if (entry === undefined) continue;
                    pieces.push(header(entry), yield* packBytes(coord.oid, rowOf(coord)));
                  }
                  return concatBytes(pieces);
                }),
              ),
            );
          }),
        );

        const r2Stream = Stream.fromIterable(r2Entries).pipe(
          Stream.flatMap((entry) =>
            Stream.succeed(header(entry)).pipe(Stream.concat(readZDataOf(entry.oid))),
          ),
        );

        return rowStream.pipe(Stream.concat(packStream), Stream.concat(r2Stream));
      }),

    // ── Store-side surface ──────────────────────────────────────────────────
    getMetaBatch,

    insertStagedBatch: Effect.fn(function* (pushId: string, objects: ReadonlyArray<StagedObject>) {
      if (objects.length === 0) return;
      const inline: Array<StagedObject> = [];
      const promoted: Array<StagedObject> = [];
      for (const object of objects) {
        if (object.pack !== undefined) {
          promoted.push(object);
        } else if (object.zdata.byteLength > R2_OFFLOAD_THRESHOLD) {
          yield* insertStagedOne(pushId, object);
        } else {
          inline.push(object);
        }
      }
      if (promoted.length > 0) {
        // Rows that point into a pack already in blob storage: no BLOB
        // written to SQLite at all. 7 bindings per row → 14 rows per
        // statement under the 100-parameter cap.
        const sorted = [...promoted].sort((a, b) => (a.oid < b.oid ? -1 : a.oid > b.oid ? 1 : 0));
        const written = yield* sql.transactionSync((raw) => {
          let changed = 0;
          for (let at = 0; at < sorted.length; at += 14) {
            const part = sorted.slice(at, at + 14);
            const values = part.map(() => "(?, ?, ?, ?, 'pack', NULL, ?, ?, ?)").join(", ");
            const bindings: Array<string | number> = [];
            for (const object of part) {
              bindings.push(
                object.oid,
                object.type,
                object.size,
                object.zsize ?? object.zdata.byteLength,
                object.pack!.packId,
                object.pack!.offset,
                pushId,
              );
            }
            changed += raw.exec(
              `INSERT OR IGNORE INTO objects (oid, type, size, zsize, location, zdata, pack_id, pack_offset, staged_push) VALUES ${values}`,
              ...bindings,
            ).rowsWritten;
          }
          return changed;
        });
        if (written < sorted.length) {
          yield* sql.inChunks(
            (ph) =>
              `UPDATE objects SET staged_push = ? WHERE oid IN (${ph})
                 AND staged_push IS NOT NULL AND staged_push != ?
                 AND staged_push NOT IN (SELECT push_id FROM pushes WHERE state = 'committed')`,
            sorted.map((object) => object.oid),
            { prefix: [pushId], suffix: [pushId] },
          );
        }
      }
      if (inline.length === 0) return;

      // One transaction for the whole batch: `INSERT OR IGNORE` needs no
      // existence probe, and a single commit replaces one per object.
      // Sorted by oid: the table is WITHOUT ROWID keyed by oid, so inserting
      // in key order turns a batch into sequential B-tree appends instead of
      // random page splits.
      const rows = inline
        .map((object) => ({
          oid: object.oid,
          type: object.type,
          size: object.size,
          zsize: object.zdata.byteLength,
          zdata: toArrayBuffer(object.zdata),
        }))
        .sort((a, b) => (a.oid < b.oid ? -1 : a.oid > b.oid ? 1 : 0));
      // Multi-row INSERTs (DESIGN §22.4): one statement per STAGE_INSERT_ROWS
      // rows instead of one per row. On production CPU each statement
      // execution costs far more than binding six more parameters, and
      // staging was the largest slice of a push's ingest (7 s of 30 s).
      const written = yield* sql.transactionSync((raw) => {
        let changed = 0;
        for (let at = 0; at < rows.length; at += STAGE_INSERT_ROWS) {
          const part = rows.slice(at, at + STAGE_INSERT_ROWS);
          const values = part.map(() => "(?, ?, ?, ?, 'row', ?, ?)").join(", ");
          const bindings: Array<string | number | ArrayBuffer> = [];
          for (const row of part) {
            bindings.push(row.oid, row.type, row.size, row.zsize, row.zdata, pushId);
          }
          const cursor = raw.exec(
            `INSERT OR IGNORE INTO objects (oid, type, size, zsize, location, zdata, staged_push) VALUES ${values}`,
            ...bindings,
          );
          changed += cursor.rowsWritten;
        }
        return changed;
      });
      // Every row was new: nothing to adopt. Only when INSERT OR IGNORE
      // skipped rows can some belong to a crashed push and need re-stamping.
      if (written >= rows.length) return;
      yield* sql.inChunks(
        (ph) =>
          `UPDATE objects SET staged_push = ? WHERE oid IN (${ph})
             AND staged_push IS NOT NULL AND staged_push != ?
             AND staged_push NOT IN (SELECT push_id FROM pushes WHERE state = 'committed')`,
        inline.map((object) => object.oid),
        { prefix: [pushId], suffix: [pushId] },
      );
    }),
    readPrepared: (pushId, oid, maxBytes) =>
      Effect.gen(function* () {
        if (!Number.isSafeInteger(maxBytes) || maxBytes < 0)
          return yield* StoreError.make({ reason: "invalid object size limit" });
        const row = yield* sql.first<ObjectMetaRow>(
          `SELECT oid, type, size, zsize, location FROM objects WHERE oid = ? AND (${LIVE_OBJECTS} OR staged_push = ?)`,
          oid,
          pushId,
        );
        if (row === undefined) return undefined;
        const meta = yield* metaFromRow(row);
        if (meta.size > maxBytes)
          return yield* StoreError.make({
            reason: `object exceeds ${maxBytes} byte read limit`,
          });
        const zdata = yield* readZBytes(oid, pushId);
        const content = yield* Zlib.inflate(zdata).pipe(Effect.mapError(zlibToStore));
        return { type: meta.type, content };
      }),

    readBase: (oid) =>
      self
        .getMeta(oid)
        .pipe(
          Effect.flatMap((meta) =>
            meta === undefined
              ? Effect.undefined
              : self.readContent(oid).pipe(Effect.map((content) => ({ type: meta.type, content }))),
          ),
        ),
    insertStaged: insertStagedOne,

    missingObjects: (oids, pushId) =>
      Effect.gen(function* () {
        const unique = Array.from(new Set(oids));
        if (unique.length === 0) {
          return [];
        }
        const rows = yield* sql.inChunks<{ oid: string }>(
          (ph) =>
            `SELECT oid FROM objects WHERE oid IN (${ph}) AND (${LIVE_OBJECTS} OR staged_push = ?)`,
          unique,
          { suffix: [pushId] },
        );
        const present = new Set(rows.map((row) => row.oid));
        return unique.filter((oid) => !present.has(oid));
      }),
  };
  return self;
};