import { BlobStore } from "@alchemy.run/git/BlobStore";
import { HasherInline, Hasher, type HasherShape } from "@alchemy.run/git/Hasher/Hasher";
import { hashObject, encodeTypeSize, makeSha1 } from "@alchemy.run/git/Protocol/ObjectCodec";
import { bufferRandomAccess } from "@alchemy.run/git/Protocol/PackParser";
import { packHeader } from "@alchemy.run/git/Protocol/PackWriter";
import * as Zlib from "@alchemy.run/git/Protocol/Zlib";
import { ingestPackFrom, ingestStoreOf } from "@alchemy.run/git/RepoObject";
import { makeObjectStore } from "@alchemy.run/git/Store/ObjectStore";
import { sliceRandomAccess } from "@alchemy.run/git/Store/PackSource";
import { makeStreamingSource } from "@alchemy.run/git/Store/StreamingSource";
import { describe, expect, test } from "@alchemy.run/test";
/**
* The hasher-driven ingest (packages/git/src/RepoObject.ts `ingestPackFrom` with a
* hasher, DESIGN §22.7) over the store harness: tiny parts force every
* delta base into an earlier part, so cross-part resolution after the end
* and thin-pack bases from the store are exercised in-process.
*/
import * as BunServices from "@effect/platform-bun/BunServices";
import { RuntimeContext } from "alchemy/RuntimeContext";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Path from "effect/Path";
import { concat } from "./harness/pack.ts";
import { makeMemoryBlobStore, makeTestSqlClient } from "./harness/store.ts";
const fixture = (name: string) =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const dir = path.join(import.meta.dirname, "fixtures", "packs");
const pack = yield* fs.readFile(path.join(dir, name));
const manifest = JSON.parse(yield* fs.readFileString(path.join(dir, "manifest.json"))) as {
packs: Record<string, { oids: ReadonlyArray<string> }>;
};
return { pack, manifest };
});
describe("ingestPackFrom through the hasher", { tags: ["unit", "local"] }, () => {
test("ofs-delta fixture in 64-byte parts: every object staged, cross-part deltas resolved after the end", async () => {
await Effect.runPromise(
Effect.gen(function* () {
const { pack, manifest } = yield* fixture("ofs-delta.pack");
const sql = makeTestSqlClient();
const store = makeObjectStore({ sql, blobs: makeMemoryBlobStore(), repoId: "R" });
const hasher = yield* Hasher;
const outcome = yield* Effect.result(
ingestPackFrom(bufferRandomAccess(pack), {
store: ingestStoreOf(store),
pushId: "push-1",
hasher,
partBytes: 64,
}),
);
if (outcome._tag === "Failure")
throw new Error(`${outcome.failure._tag}: ${outcome.failure.reason}`);
const result = outcome.success;
const expected = manifest.packs["ofs-delta"]!.oids;
expect(result.objectCount).toBe(expected.length);
const staged = yield* sql.all<{ oid: string }>(
`SELECT oid FROM objects WHERE staged_push = 'push-1' ORDER BY oid`,
);
expect(staged.map((r) => r.oid)).toEqual([...expected].sort());
}).pipe(
Effect.provide(
HasherInline.pipe(Layer.provide(Layer.succeed(BlobStore, makeMemoryBlobStore()))),
),
Effect.provide(BunServices.layer),
),
);
});
test("a corrupted trailer is rejected by the pipeline's lagged hash", async () => {
await Effect.runPromise(
Effect.gen(function* () {
const { pack } = yield* fixture("simple.pack");
const bad = Uint8Array.from(pack);
bad[bad.length - 3] ^= 0x01;
const store = makeObjectStore({
sql: makeTestSqlClient(),
blobs: makeMemoryBlobStore(),
repoId: "R",
});
const hasher = yield* Hasher;
const r = yield* Effect.result(
ingestPackFrom(bufferRandomAccess(bad), {
store: ingestStoreOf(store),
pushId: "p",
hasher,
partBytes: 4096,
}),
);
expect(r._tag).toBe("Failure");
if (r._tag === "Failure") expect(r.failure.reason).toContain("checksum");
}).pipe(
Effect.provide(
HasherInline.pipe(Layer.provide(Layer.succeed(BlobStore, makeMemoryBlobStore()))),
),
Effect.provide(BunServices.layer),
),
);
});
});
describe(
"hasher pipeline over a streaming source with eviction",
{ tags: ["unit", "local"] },
() => {
test(
"a multi-part pack whose retained window is smaller than a part stages every inline row and completes (no fallback)",
async () => {
await Effect.runPromise(
Effect.gen(function* () {
const n = 1500;
const pieces: Array<Uint8Array> = [packHeader(n)];
for (let i = 0; i < n; i++) {
const c = new Uint8Array(1000 + (i % 700));
crypto.getRandomValues(c);
yield* hashObject(3, c);
pieces.push(encodeTypeSize(3, c.length), yield* Zlib.deflate(c));
}
const body = concat(pieces);
const sha = makeSha1();
sha.update(body);
const pack = concat([body, sha.digest()]);
// Parts of 64 KiB, retention of 128 KiB, no spill/fallback: every
// inline row must be read back before its bytes are dropped.
const feeder = makeStreamingSource({
slabBytes: 32 * 1024,
retainBytes: 128 * 1024,
backpressureBytes: 1 << 20,
});
const sql = makeTestSqlClient();
const store = makeObjectStore({ sql, blobs: makeMemoryBlobStore(), repoId: "R" });
const hasher = yield* Hasher;
const ingest = yield* Effect.forkChild(
Effect.result(
ingestPackFrom(feeder.source, {
store: ingestStoreOf(store),
pushId: "p",
hasher,
partBytes: 64 * 1024,
}),
),
);
for (let at = 0; at < pack.length; at += 50_000)
yield* feeder.push(pack.subarray(at, at + 50_000));
feeder.end();
const r = yield* Fiber.join(ingest).pipe(Effect.timeout("20 seconds"));
expect(r._tag).toBe("Success");
const staged = yield* sql.first<{ n: number; withBytes: number }>(
`SELECT COUNT(*) AS n, SUM(LENGTH(zdata) > 0) AS withBytes FROM objects WHERE staged_push = 'p'`,
);
expect(staged?.n).toBe(n);
expect(staged?.withBytes).toBe(n);
}).pipe(
Effect.provide(
HasherInline.pipe(Layer.provide(Layer.succeed(BlobStore, makeMemoryBlobStore()))),
),
),
);
},
{ timeout: 30_000 },
);
},
);
describe(
"raw-chunk dispatch with resync and stitching (DESIGN §22.9)",
{ tags: ["unit", "local"] },
() => {
test("the ofs-delta fixture in 64-byte chunks: every straddler stitched, every delta resolved", async () => {
await Effect.runPromise(
Effect.gen(function* () {
const { pack, manifest } = yield* fixture("ofs-delta.pack");
const sql = makeTestSqlClient();
const store = makeObjectStore({ sql, blobs: makeMemoryBlobStore(), repoId: "R" });
const hasher = yield* Hasher;
for (const partBytes of [64, 200, 1000, 1 << 20]) {
yield* sql.run(`DELETE FROM objects`);
const outcome = yield* Effect.result(
ingestPackFrom(bufferRandomAccess(pack), {
store: ingestStoreOf(store),
pushId: "p",
hasher,
partBytes,
}),
);
if (outcome._tag === "Failure")
throw new Error(
`parts of ${partBytes}: ${outcome.failure._tag}: ${outcome.failure.reason}`,
);
const result = outcome.success;
const expected = manifest.packs["ofs-delta"]!.oids;
expect(result.objectCount, `parts of ${partBytes}`).toBe(expected.length);
const staged = yield* sql.all<{ oid: string }>(
`SELECT oid FROM objects WHERE staged_push = 'p' ORDER BY oid`,
);
expect(
staged.map((r) => r.oid),
`parts of ${partBytes}`,
).toEqual([...expected].sort());
}
}).pipe(
Effect.provide(
HasherInline.pipe(Layer.provide(Layer.succeed(BlobStore, makeMemoryBlobStore()))),
),
Effect.provide(BunServices.layer),
),
);
});
},
);
describe("spill by the hasher (DESIGN §22.10)", { tags: ["unit", "local"] }, () => {
/** A pack of `n` random blobs plus a fake command section in front. */
const makeBody = (n: number, blobBytes: number, headBytes: number) =>
Effect.gen(function* () {
const pieces: Array<Uint8Array> = [packHeader(n)];
for (let i = 0; i < n; i++) {
const c = new Uint8Array(blobBytes + (i % 97));
crypto.getRandomValues(c);
pieces.push(encodeTypeSize(3, c.length), yield* Zlib.deflate(c));
}
const packBody = concat(pieces);
const sha = makeSha1();
sha.update(packBody);
const pack = concat([packBody, sha.digest()]);
const head = new Uint8Array(headBytes);
head.fill(0x20);
return { body: concat([head, pack]), packStart: headBytes };
});
const run = (
opts: {
readonly n: number;
readonly blobBytes: number;
readonly headBytes: number;
readonly partBytes: number;
readonly threshold: number;
readonly chunk: number;
},
hasherOverride?: HasherShape,
) =>
Effect.gen(function* () {
const { body, packStart } = yield* makeBody(opts.n, opts.blobBytes, opts.headBytes);
// ONE blob store: the hasher writes the parts the pump's upload
// collects — in production both are the same BlobStore layer.
const blobs = yield* BlobStore;
const feeder = makeStreamingSource({
slabBytes: opts.partBytes,
retainBytes: opts.partBytes * 2,
backpressureBytes: 1 << 20,
});
const sql = makeTestSqlClient();
const store = makeObjectStore({ sql, blobs, repoId: "R" });
const hasher = hasherOverride ?? (yield* Hasher);
const source = sliceRandomAccess(feeder.source, packStart);
const ingest = yield* Effect.forkChild(
Effect.result(
ingestPackFrom(source, {
store: ingestStoreOf(store),
pushId: "p",
hasher,
partBytes: opts.partBytes,
spill: {
body: feeder.source,
feeder,
packStart,
blobs,
key: "R/incoming/X.pack",
packId: "wire-X",
repoId: "R",
threshold: opts.threshold,
},
}),
),
);
for (let at = 0; at < body.length; at += opts.chunk)
yield* feeder.push(body.subarray(at, at + opts.chunk));
feeder.end();
const r = yield* Fiber.join(ingest).pipe(Effect.timeout("20 seconds"));
if (r._tag === "Failure") throw new Error(r.failure.reason);
const rows = yield* sql.first<{ n: number; promoted: number }>(
`SELECT COUNT(*) AS n, SUM(location = 'pack') AS promoted FROM objects WHERE staged_push = 'p'`,
);
const spilled = yield* blobs.head("R/incoming/X.pack");
return { result: r.success, rows, spilled, body, blobs };
}).pipe(
Effect.provide(
Layer.provideMerge(HasherInline, Layer.succeed(BlobStore, makeMemoryBlobStore())),
),
Effect.provide(RuntimeContext.phantom),
);
test(
"a body past the threshold is written whole by the parts' hashers; blobs are promoted into it",
async () => {
const out = await Effect.runPromise(
run({
n: 400,
blobBytes: 600,
headBytes: 137,
partBytes: 32 * 1024,
threshold: 8 * 1024,
chunk: 7_001,
}),
);
expect(out.result.parkedKey).toBe("R/incoming/X.pack");
expect(out.spilled?.size).toBe(out.body.length);
const stored = await Effect.runPromise(
Effect.gen(function* () {
const blob = yield* out.blobs.get("R/incoming/X.pack");
return blob === null ? undefined : yield* blob.bytes;
}).pipe(Effect.provide(RuntimeContext.phantom)),
);
expect(stored).toEqual(out.body);
expect(out.rows?.n).toBe(400);
expect(out.rows?.promoted).toBe(400);
expect(out.result.promoted).toBe(400);
},
{ timeout: 30_000 },
);
test(
"a hasher that cannot write the spill (writesSpill=false, small chunks): the pump uploads the parts itself",
async () => {
const out = await Effect.runPromise(
Effect.gen(function* () {
const inline = yield* Hasher;
// A Lambda-shaped hasher: chunks a quarter of a part, no spill.
const lambdaLike: HasherShape = {
writesSpill: false,
chunkBytes: 8 * 1024,
hashPart: (payload, opts) => {
if (opts.spill !== undefined) {
throw new Error("spill must not be requested from a non-spilling hasher");
}
return inline.hashPart(payload, opts);
},
hashBoundsPart: inline.hashBoundsPart,
resolveDeltas: inline.resolveDeltas,
};
return yield* run(
{
n: 400,
blobBytes: 600,
headBytes: 137,
partBytes: 32 * 1024,
threshold: 8 * 1024,
chunk: 7_001,
},
lambdaLike,
);
}).pipe(
Effect.provide(
Layer.provideMerge(HasherInline, Layer.succeed(BlobStore, makeMemoryBlobStore())),
),
Effect.provide(RuntimeContext.phantom),
),
);
expect(out.result.parkedKey).toBe("R/incoming/X.pack");
expect(out.spilled?.size).toBe(out.body.length);
const stored = await Effect.runPromise(
Effect.gen(function* () {
const blob = yield* out.blobs.get("R/incoming/X.pack");
return blob === null ? undefined : yield* blob.bytes;
}).pipe(Effect.provide(RuntimeContext.phantom)),
);
expect(stored).toEqual(out.body);
expect(out.rows?.n).toBe(400);
expect(out.rows?.promoted).toBe(400);
},
{ timeout: 30_000 },
);
test(
"a body that ends within the threshold stays in memory: nothing written, rows inline",
async () => {
const out = await Effect.runPromise(
run({
n: 20,
blobBytes: 100,
headBytes: 50,
partBytes: 32 * 1024,
threshold: 64 * 1024,
chunk: 1_000,
}),
);
expect(out.result.parkedKey).toBeUndefined();
expect(out.spilled).toBeNull();
expect(out.rows?.n).toBe(20);
expect(out.rows?.promoted).toBe(0);
},
{ timeout: 30_000 },
);
});