sailfork / t3code-github / .repos/ alchemy-effect/ packages/ integration-tests/ test/ Git/ store.test.ts

import {
  runCompactJob,
  runGeometricMergeJob,
  shouldCompact,
  BLOB_TYPE,
} from "@alchemy.run/git/Jobs/Compact";
/**
 * Object store unit tests (packages/git/src/Store/ObjectStore.ts, Jobs/Compact.ts)
 * over the bun:sqlite + in-memory blob harness. Every emitted pack is
 * re-parsed by the real pack parser, so the bytes — not just the counts —
 * are verified.
 */
import {
  encodeTypeSize,
  makeSha1,
  hashObject,
  type Oid,
  type ObjectType,
} from "@alchemy.run/git/Protocol/ObjectCodec";
import { bufferRandomAccess, ingestPack } from "@alchemy.run/git/Protocol/PackParser";
import { packHeader } from "@alchemy.run/git/Protocol/PackWriter";
import type { ManifestEntry } from "@alchemy.run/git/Protocol/Store";
import * as Zlib from "@alchemy.run/git/Protocol/Zlib";
import { packKey, packKeyOf, wirePackId } from "@alchemy.run/git/Store/Keys";
import {
  makeObjectStore,
  STAGE_INSERT_ROWS,
  WINDOW_BYTES,
  WINDOW_CACHE_BYTES,
} from "@alchemy.run/git/Store/ObjectStore";
import { describe, expect, test } from "@alchemy.run/test";
import { RuntimeContext } from "alchemy/RuntimeContext";
import * as Effect from "effect/Effect";
import * as Stream from "effect/Stream";
import { concat, verifyPack } from "./harness/pack.ts";
import { makeMemoryBlobStore, makeTestSqlClient } from "./harness/store.ts";

const REPO = "01TESTREPO0000000000000000";

/** Runs an effect, surfacing typed failures with their fields (not just `message`). */
const run = <A>(effect: Effect.Effect<A, unknown, RuntimeContext>) =>
  Effect.runPromise(
    effect.pipe(
      Effect.provide(RuntimeContext.phantom),
      Effect.mapError((e) =>
        e instanceof Error && !("reason" in e) ? e : new Error(JSON.stringify(e)),
      ),
    ),
  );

/** Incompressible bytes; `seed` only guarantees distinct content per call. */
const bytes = (n: number, seed: number) => {
  const out = new Uint8Array(n);
  for (let at = 0; at < n; at += 65536) {
    crypto.getRandomValues(out.subarray(at, Math.min(at + 65536, n)));
  }
  out[0] = seed & 0xff;
  return out;
};

interface Fixture {
  readonly oid: Oid;
  readonly type: ObjectType;
  readonly content: Uint8Array;
  readonly zdata: Uint8Array;
}

const makeFixture = (type: ObjectType, content: Uint8Array) =>
  Effect.gen(function* () {
    const oid = yield* hashObject(type, content);
    const zdata = yield* Zlib.deflate(content);
    return { oid, type, content, zdata } satisfies Fixture;
  });

/** A repo with `blobs` incompressible blobs of `blobSize` and `trees` small trees. */
const seedRepo = (options: { blobs: number; blobSize: number; trees: number }) =>
  Effect.gen(function* () {
    const sql = makeTestSqlClient();
    const blobs = makeMemoryBlobStore();
    const store = makeObjectStore({ sql, blobs, repoId: REPO });
    const fixtures: Array<Fixture> = [];
    for (let i = 0; i < options.blobs; i++) {
      fixtures.push(yield* makeFixture(3, bytes(options.blobSize, i + 1)));
    }
    for (let i = 0; i < options.trees; i++) {
      // Not a real tree encoding; the store never parses tree bytes here.
      fixtures.push(yield* makeFixture(2, new TextEncoder().encode(`tree ${i} ${"x".repeat(40)}`)));
    }
    yield* sql.transactionSync((raw) => {
      for (const f of fixtures) {
        raw.exec(
          `INSERT INTO objects (oid, type, size, zsize, location, zdata) VALUES (?, ?, ?, ?, 'row', ?)`,
          f.oid,
          f.type,
          f.content.length,
          f.zdata.length,
          f.zdata.buffer.slice(f.zdata.byteOffset, f.zdata.byteOffset + f.zdata.byteLength),
        );
      }
    });
    return { sql, blobs, store, fixtures };
  });

const manifestOf = (fixtures: ReadonlyArray<Fixture>, location: ManifestEntry["location"]) =>
  fixtures.map((f): ManifestEntry => ({
    oid: f.oid,
    type: f.type,
    size: f.content.length,
    zsize: f.zdata.length,
    location,
  }));

/** Emits a full pack (header + entries + trailer) through `packEntries`. */
const buildPack = (
  store: ReturnType<typeof makeObjectStore>,
  entries: ReadonlyArray<ManifestEntry>,
) =>
  Effect.gen(function* () {
    const body = Array.from(yield* Stream.runCollect(store.packEntries(entries)));
    const head = packHeader(entries.length);
    const sha = makeSha1();
    sha.update(head);
    for (const b of body) sha.update(b);
    return concat([head, ...body, sha.digest()]);
  });

/** Parses a pack with the real parser and returns the oids it yields. */
const parsePack = (pack: Uint8Array, store: ReturnType<typeof makeObjectStore>) =>
  Effect.gen(function* () {
    const seen = new Map<string, number>();
    const summary = yield* ingestPack({
      source: bufferRandomAccess(pack),
      store,
      sink: (entry) =>
        Effect.sync(() => {
          seen.set(entry.oid, entry.size);
        }),
    });
    return { count: summary.count, seen };
  });

const compactAll = (
  sql: ReturnType<typeof makeTestSqlClient>,
  blobs: ReturnType<typeof makeMemoryBlobStore>,
  maxObjects = 100,
) =>
  Effect.gen(function* () {
    const packIds: Array<string> = [];
    for (;;) {
      const out = yield* runCompactJob({
        repoId: REPO,
        sql,
        blobs,
        maxObjects,
        maxBytes: 64 * 1024 * 1024,
      });
      if (out.packId !== undefined) packIds.push(out.packId);
      if (!out.more) break;
    }
    return packIds;
  });

describe("compaction is blob-only", { tags: ["unit", "local"] }, () => {
  test("shouldCompact ignores commits/trees; runCompactJob leaves them as rows", async () => {
    await run(
      Effect.gen(function* () {
        const { sql, blobs, fixtures } = yield* seedRepo({ blobs: 5, blobSize: 100, trees: 50 });
        expect(BLOB_TYPE).toBe(3);
        // 55 rows total, but only 5 blobs: below a count threshold of 10.
        expect(yield* shouldCompact(sql, { countThreshold: 10, bytesThreshold: 1 << 30 })).toBe(
          false,
        );
        expect(yield* shouldCompact(sql, { countThreshold: 5, bytesThreshold: 1 << 30 })).toBe(
          true,
        );
        const packs = yield* compactAll(sql, blobs);
        expect(packs.length).toBe(1);
        const rows = yield* sql.all<{ location: string; type: number; n: number }>(
          `SELECT location, type, COUNT(*) AS n FROM objects GROUP BY location, type ORDER BY location, type`,
        );
        expect(rows).toEqual([
          { location: "pack", type: 3, n: 5 },
          { location: "row", type: 2, n: 50 },
        ]);
        expect(yield* runCompactJob({ repoId: REPO, sql, blobs })).toMatchObject({
          moved: 0,
          more: false,
        });
        expect(fixtures.length).toBe(55);
      }),
    );
  });
});

describe("packEntries", { tags: ["unit", "local"] }, () => {
  test(
    "rows + multi-window pack (with straddling objects) emit a pack the parser accepts",
    async () => {
      await run(
        Effect.gen(function* () {
          // 300 × 30 KiB incompressible ≈ 9 MiB → 3 windows, with objects
          // straddling both window edges.
          const { sql, blobs, store, fixtures } = yield* seedRepo({
            blobs: 300,
            blobSize: 30 * 1024,
            trees: 40,
          });
          const packs = yield* compactAll(sql, blobs, 1000); // one pack
          expect(packs.length).toBe(1);
          const packBytes = blobs.objects.get(packKey(REPO, packs[0]!))!;
          expect(packBytes.length).toBeGreaterThan(2 * WINDOW_BYTES); // 3 windows
          const coords = yield* sql.all<{ pack_id: string; n: number }>(
            `SELECT pack_id, COUNT(*) AS n FROM objects WHERE location='pack' GROUP BY pack_id`,
          );
          expect(coords.map((c) => c.n)).toEqual([300]);

          // Manifest in the order a closure would produce (trees first).
          const entries = [
            ...manifestOf(
              fixtures.filter((f) => f.type === 2),
              "row",
            ),
            ...manifestOf(
              fixtures.filter((f) => f.type === 3),
              "pack",
            ),
          ];
          blobs.gets.length = 0;
          const pack = yield* buildPack(store, entries);
          expect(verifyPack(pack).error).toBeUndefined();
          const parsed = yield* parsePack(pack, store);
          expect(parsed.count).toBe(340);
          expect(parsed.seen.size).toBe(340);
          for (const f of fixtures) expect(parsed.seen.get(f.oid)).toBe(f.content.length);

          // One ranged GET per window touched, not per object.
          const windowGets = blobs.gets.filter((g) => g.length === WINDOW_BYTES);
          const straddleGets = blobs.gets.filter((g) => g.length !== WINDOW_BYTES);
          expect(windowGets.length).toBe(3);
          expect(straddleGets.length).toBeLessThanOrEqual(2); // ≤ one per window edge
        }),
      );
    },
    { timeout: 60_000 },
  );

  test("a manifest entry that compaction moved after the closure is still emitted (re-dispatch)", async () => {
    await run(
      Effect.gen(function* () {
        const { sql, blobs, store, fixtures } = yield* seedRepo({
          blobs: 20,
          blobSize: 2048,
          trees: 3,
        });
        // The closure saw everything as rows…
        const entries = manifestOf(fixtures, "row");
        // …then compaction ran before emission.
        yield* compactAll(sql, blobs);
        const pack = yield* buildPack(store, entries);
        expect(verifyPack(pack).error).toBeUndefined();
        const parsed = yield* parsePack(pack, store);
        expect(parsed.seen.size).toBe(23);
      }),
    );
  });

  test("a truly absent object fails the stream instead of emitting a short pack", async () => {
    await run(
      Effect.gen(function* () {
        const { store, fixtures } = yield* seedRepo({ blobs: 2, blobSize: 100, trees: 0 });
        const ghost = { ...manifestOf(fixtures, "row")[0]!, oid: "0".repeat(40) as Oid };
        const result = yield* Effect.result(Stream.runCollect(store.packEntries([ghost])));
        expect(result._tag).toBe("Failure");
      }),
    );
  });
});

describe("readContentBatch", { tags: ["unit", "local"] }, () => {
  test("returns inflated content for rows and packed objects alike", async () => {
    await run(
      Effect.gen(function* () {
        const { sql, blobs, store, fixtures } = yield* seedRepo({
          blobs: 30,
          blobSize: 512,
          trees: 10,
        });
        yield* compactAll(sql, blobs);
        const out = yield* store.readContentBatch(fixtures.map((f) => f.oid));
        expect(out.size).toBe(40);
        for (const f of fixtures) {
          expect(Array.from(out.get(f.oid)!)).toEqual(Array.from(f.content));
        }
      }),
    );
  });
});

describe("window cache", { tags: ["unit", "local"] }, () => {
  test(
    "is bounded by WINDOW_CACHE_BYTES: re-reading the first of nine windows refetches",
    async () => {
      await run(
        Effect.gen(function* () {
          // 36 × 1 MiB incompressible → one 36 MiB pack = 9 windows > 8 cached.
          const { sql, blobs, store, fixtures } = yield* seedRepo({
            blobs: 36,
            blobSize: 1024 * 1024,
            trees: 0,
          });
          const out = yield* runCompactJob({
            repoId: REPO,
            sql,
            blobs,
            maxObjects: 100,
            maxBytes: 64 * 1024 * 1024,
          });
          expect(out.moved).toBe(36);
          expect(WINDOW_CACHE_BYTES / WINDOW_BYTES).toBe(8);
          const sorted = [...fixtures];
          const order = yield* sql.all<{ oid: string; pack_offset: number; zsize: number }>(
            `SELECT oid, pack_offset, zsize FROM objects ORDER BY pack_offset`,
          );
          const byOid = new Map(sorted.map((f) => [f.oid, f]));
          const first = byOid.get(order[0]!.oid as Oid)!;
          // An object lying entirely inside window 7 (objects that straddle a
          // window edge take a ranged read that bypasses the cache by design).
          const inside7 = order.find(
            (r) =>
              Math.floor(r.pack_offset / WINDOW_BYTES) === 7 &&
              r.pack_offset + r.zsize <= 8 * WINDOW_BYTES,
          )!;
          const last = byOid.get(inside7.oid as Oid)!;
          blobs.gets.length = 0;
          yield* store.readContentBatch([first.oid]);
          const afterFirst = blobs.gets.length;
          expect(afterFirst).toBeGreaterThanOrEqual(1);
          // Touch every window.
          yield* store.readContentBatch(order.map((r) => r.oid as Oid));
          blobs.gets.length = 0;
          // The first window has been evicted (9 > 8): reading it fetches again.
          yield* store.readContentBatch([first.oid]);
          expect(blobs.gets.length).toBeGreaterThanOrEqual(1);
          // The last window is still hot: no fetch.
          blobs.gets.length = 0;
          yield* store.readContentBatch([last.oid]);
          expect(blobs.gets.length).toBe(0);
        }),
      );
    },
    { timeout: 60_000 },
  );
});

describe("insertStagedBatch", { tags: ["unit", "local"] }, () => {
  test("stages a batch with multi-row inserts; staged rows are invisible to live reads", async () => {
    await run(
      Effect.gen(function* () {
        const sql = makeTestSqlClient();
        const store = makeObjectStore({ sql, blobs: makeMemoryBlobStore(), repoId: REPO });
        const objects = [];
        for (let i = 0; i < STAGE_INSERT_ROWS * 2 + 7; i++) {
          const f = yield* makeFixture(3, bytes(64, i + 1));
          objects.push({ oid: f.oid, type: f.type, size: f.content.length, zdata: f.zdata });
        }
        yield* store.insertStagedBatch("push-A", objects);
        const staged = yield* sql.first<{ n: number }>(
          `SELECT COUNT(*) AS n FROM objects WHERE staged_push = 'push-A'`,
        );
        expect(staged?.n).toBe(objects.length);
        expect(yield* store.has(objects[0]!.oid)).toBe(false);
      }),
    );
  });

  test("a committed push's rows are live before the flip, cannot be adopted, and GC flips them (DESIGN §22.14)", async () => {
    await run(
      Effect.gen(function* () {
        const sql = makeTestSqlClient();
        const store = makeObjectStore({ sql, blobs: makeMemoryBlobStore(), repoId: REPO });
        const objects = [];
        for (let i = 0; i < 6; i++) {
          const f = yield* makeFixture(3, bytes(64, i + 300));
          objects.push({ oid: f.oid, type: f.type, size: f.content.length, zdata: f.zdata });
        }
        yield* sql.run(
          `INSERT INTO pushes (push_id, started_at, state) VALUES ('push-live', ?, 'committed')`,
          Date.now(),
        );
        yield* store.insertStagedBatch("push-live", objects);
        // Live for readers although `staged_push` is still set.
        expect(yield* store.has(objects[0]!.oid)).toBe(true);
        const missing = yield* store.missingObjects(
          objects.map((o) => o.oid),
          "push-other",
        );
        expect(missing).toEqual([]);
        // A later push re-staging the same oids must not steal them.
        yield* store.insertStagedBatch("push-other", objects.slice(0, 3));
        const still = yield* sql.first<{ n: number }>(
          `SELECT COUNT(*) AS n FROM objects WHERE staged_push = 'push-live'`,
        );
        expect(still?.n).toBe(6);
        // The deferred flip (here: what GC does) nulls the column.
        yield* sql.run(
          `UPDATE objects SET staged_push = NULL WHERE staged_push IN (SELECT push_id FROM pushes WHERE state = 'committed')`,
        );
        const flipped = yield* sql.first<{ n: number }>(
          `SELECT COUNT(*) AS n FROM objects WHERE staged_push IS NULL`,
        );
        expect(flipped?.n).toBe(6);
        expect(yield* store.has(objects[5]!.oid)).toBe(true);
      }),
    );
  });

  test("rows left by a crashed push are adopted; live objects are left alone", async () => {
    await run(
      Effect.gen(function* () {
        const sql = makeTestSqlClient();
        const store = makeObjectStore({ sql, blobs: makeMemoryBlobStore(), repoId: REPO });
        const objects = [];
        for (let i = 0; i < 10; i++) {
          const f = yield* makeFixture(3, bytes(64, i + 100));
          objects.push({ oid: f.oid, type: f.type, size: f.content.length, zdata: f.zdata });
        }
        yield* store.insertStagedBatch("push-crashed", objects.slice(0, 4));
        yield* store.insertStagedBatch("push-B", objects);
        const byPush = yield* sql.all<{ staged_push: string; n: number }>(
          `SELECT staged_push, COUNT(*) AS n FROM objects GROUP BY staged_push`,
        );
        expect(byPush).toEqual([{ staged_push: "push-B", n: 10 }]);
        yield* sql.run(`UPDATE objects SET staged_push = NULL WHERE oid = ?`, objects[0]!.oid);
        yield* store.insertStagedBatch("push-C", objects.slice(0, 2));
        const live = yield* sql.first<{ staged_push: string | null }>(
          `SELECT staged_push FROM objects WHERE oid = ?`,
          objects[0]!.oid,
        );
        expect(live?.staged_push).toBeNull();
        const other = yield* sql.first<{ staged_push: string | null }>(
          `SELECT staged_push FROM objects WHERE oid = ?`,
          objects[1]!.oid,
        );
        expect(other?.staged_push).toBe("push-C");
      }),
    );
  });
});

describe("promoted wire packs (DESIGN §22.5)", { tags: ["unit", "local"] }, () => {
  /** Lays fixtures out the way a wire pack does: typeSize header + zdata per entry. */
  const layout = (fixtures: ReadonlyArray<Fixture>, base: number) => {
    const pieces: Array<Uint8Array> = [new Uint8Array(base)];
    const offsets = new Map<Oid, number>();
    let at = base;
    for (const f of fixtures) {
      const head = encodeTypeSize(f.type as 3, f.content.length);
      pieces.push(head, f.zdata);
      offsets.set(f.oid, at + head.length);
      at += head.length + f.zdata.length;
    }
    return { bytes: concat(pieces), offsets };
  };

  test("rows staged as pack references read back through the wire key on every path", async () => {
    await run(
      Effect.gen(function* () {
        const sql = makeTestSqlClient();
        const blobs = makeMemoryBlobStore();
        const store = makeObjectStore({ sql, blobs, repoId: REPO });
        const fixtures: Array<Fixture> = [];
        for (let i = 0; i < 40; i++) fixtures.push(yield* makeFixture(3, bytes(3000, i + 1)));
        const packId = wirePackId("01RECEIVE00000000000000000");
        const { bytes: wire, offsets } = layout(fixtures, 137); // 137 bytes of pkt-line commands first
        yield* blobs.put(packKeyOf(REPO, packId), wire);
        expect(packKeyOf(REPO, packId)).toContain("/incoming/");
        yield* store.insertStagedBatch(
          "push-P",
          fixtures.map((f) => ({
            oid: f.oid,
            type: f.type,
            size: f.content.length,
            zdata: f.zdata,
            pack: { packId, offset: offsets.get(f.oid)! },
          })),
        );
        const staged = yield* sql.all<{ location: string; n: number; withBlob: number }>(
          `SELECT location, COUNT(*) AS n, SUM(zdata IS NOT NULL) AS withBlob FROM objects GROUP BY location`,
        );
        expect(staged).toEqual([{ location: "pack", n: 40, withBlob: 0 }]);
        yield* sql.run(`UPDATE objects SET staged_push = NULL WHERE staged_push = 'push-P'`);
        // Single reads, batched reads, and pack emission all resolve the wire key.
        const one = yield* store.readContent(fixtures[7]!.oid);
        expect(Array.from(one)).toEqual(Array.from(fixtures[7]!.content));
        const many = yield* store.readContentBatch(fixtures.map((f) => f.oid));
        expect(many.size).toBe(40);
        const pack = yield* buildPack(store, manifestOf(fixtures, "pack"));
        expect(verifyPack(pack).error).toBeUndefined();
        const parsed = yield* parsePack(
          pack,
          makeObjectStore({ sql: makeTestSqlClient(), blobs: makeMemoryBlobStore(), repoId: "X" }),
        );
        expect(parsed.seen.size).toBe(40);
      }),
    );
  });

  test("compaction leaves promoted rows alone; geometric merge REWRITES a wire pack into the merged pack", async () => {
    await run(
      Effect.gen(function* () {
        const { sql, blobs, store, fixtures } = yield* seedRepo({
          blobs: 30,
          blobSize: 2000,
          trees: 3,
        });
        const promotedFixtures: Array<Fixture> = [];
        for (let i = 0; i < 12; i++)
          promotedFixtures.push(yield* makeFixture(3, bytes(2000, i + 500)));
        const packId = wirePackId("01RECEIVE00000000000000001");
        // The wire object carries junk around the referenced spans (a real
        // one carries trees, commits and delta entries): only spans move.
        const { bytes: wire, offsets } = layout(promotedFixtures, 333);
        const withJunk = concat([wire, bytes(5000, 9)]);
        yield* blobs.put(packKeyOf(REPO, packId), withJunk);
        yield* store.insertStagedBatch(
          "push-Q",
          promotedFixtures.map((f) => ({
            oid: f.oid,
            type: f.type,
            size: f.content.length,
            zdata: f.zdata,
            pack: { packId, offset: offsets.get(f.oid)! },
          })),
        );
        yield* sql.run(`UPDATE objects SET staged_push = NULL`);
        const p1 = yield* runCompactJob({
          repoId: REPO,
          sql,
          blobs,
          maxObjects: 15,
          maxBytes: 1 << 30,
        });
        const p2 = yield* runCompactJob({
          repoId: REPO,
          sql,
          blobs,
          maxObjects: 15,
          maxBytes: 1 << 30,
        });
        expect(p1.moved + p2.moved).toBe(30);
        const merge = yield* runGeometricMergeJob({ repoId: REPO, sql, blobs });
        expect(merge.packs).toBe(3);
        expect(merge.pendingDelete).toContain(packKeyOf(REPO, packId));
        const packs = yield* sql.all<{ pack_id: string; n: number }>(
          `SELECT pack_id, COUNT(*) AS n FROM objects WHERE location = 'pack' GROUP BY pack_id`,
        );
        expect(packs).toEqual([{ pack_id: merge.packId, n: 42 }]);
        // The merged pack is a clean packfile the parser accepts, and every
        // object — promoted ones included — reads back through it.
        const merged = blobs.objects.get(packKey(REPO, merge.packId!))!;
        expect(verifyPack(merged)).toEqual({ objects: 42 });
        const all = yield* store.readContentBatch(
          [...fixtures, ...promotedFixtures].map((f) => f.oid),
        );
        expect(all.size).toBe(45);
        for (const f of promotedFixtures)
          expect(Array.from(all.get(f.oid)!)).toEqual(Array.from(f.content));
      }),
    );
  });
});

describe("prepared object view", { tags: ["unit", "local"] }, () => {
  test("reads only its own staged objects and live objects, with a size cap", () =>
    run(
      Effect.gen(function* () {
        const sql = makeTestSqlClient();
        const store = makeObjectStore({ sql, blobs: makeMemoryBlobStore(), repoId: REPO });
        const own = yield* makeFixture(3, new TextEncoder().encode("own"));
        const other = yield* makeFixture(3, new TextEncoder().encode("other"));
        yield* store.insertStaged("mine", { ...own, size: own.content.length });
        yield* store.insertStaged("theirs", { ...other, size: other.content.length });
        expect(yield* store.getMeta(own.oid)).toBeUndefined();
        expect((yield* store.readPrepared("mine", own.oid, 3))?.content).toEqual(own.content);
        expect(yield* store.readPrepared("mine", other.oid, 100)).toBeUndefined();
        const tooBig = yield* Effect.result(store.readPrepared("mine", own.oid, 2));
        expect(tooBig._tag).toBe("Failure");
      }),
    ));
});