/**
* The per-repo Durable Object (DESIGN.md §2, §3.4, §3.6, §8).
*
* One DO per repo, addressed by `getByName(repoId)` (a ULID — never
* `owner/name`, so renames never move data). The DO owns refs, the object
* index and loose bytes, tokens, the commit graph, push staging, and the
* alarm jobs (import / fork / delete-purge / staging-GC). It runs the git
* v0 wire protocol in its `fetch` handler — advertisement, upload-pack,
* receive-pack — because every byte those need lives in this DO's SQLite.
*
* NOTE: the protocol *choreography* (advertisement builder, request
* parsers, pack writer/parser) is implemented in this file against the
* pure codec primitives in `src/Protocol/` (`Pkt`, `Sideband`, `ObjectCodec`,
* `Delta`, `Zlib`, `Store`). DESIGN.md places these in
* `src/git/{Advertise,UploadPack,ReceivePack,PackParser,PackWriter}.ts`;
* they live here (exported) until the git-layer split lands — moving them
* is a mechanical extraction.
*
* ### Wire endpoints
* The Worker proxies these untouched to `stub.fetch`:
* **Example:** Routes served by this DO's fetch handler
* ```
* GET /:owner/:repo[.git]/info/refs?service=git-(upload|receive)-pack
* POST /:owner/:repo[.git]/git-upload-pack
* POST /:owner/:repo[.git]/git-receive-pack
* ```
*/
import * as Cloudflare from "@alchemy.run/cloudflare";
import type * as cf from "@cloudflare/workers-types";
import type { HttpEffect } from "alchemy/Http";
import { RuntimeContext } from "alchemy/RuntimeContext";
import * as Context from "effect/Context";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as FetchHttpClient from "effect/http/FetchHttpClient";
import * as HttpServerRequest from "effect/http/HttpServerRequest";
import * as HttpServerResponse from "effect/http/HttpServerResponse";
import * as Option from "effect/Option";
import * as Result from "effect/Result";
import * as Schema from "effect/Schema";
import * as Semaphore from "effect/Semaphore";
import * as Stream from "effect/Stream";
import {
BranchMissing,
MergeConflict,
NoMergeBase,
NothingToMerge,
ObjectNotFound,
PullExists,
PullNotFound,
PullStateConflict,
ReadOnlyRepo,
RefConflict,
RefNotFound,
RepoNotFound,
ValidationError,
WrongObjectType,
type Oid as ApiOid,
type RepoStatus,
} from "./Api.ts";
import type { BlobMultipart, UploadedPart } from "./BlobStore.ts";
import { BlobStore, type BlobStoreShape } from "./BlobStore.ts";
import { type HasherShape, type HashPartResult } from "./Hasher/Hasher.ts";
import { bundleCovers, runBundleJob, type BundleInfo } from "./Jobs/Bundle.ts";
import { runCompactJob, runGeometricMergeJob, shouldCompact } from "./Jobs/Compact.ts";
import { runForkJob, snapshotStream, type SnapshotChunk } from "./Jobs/Fork.ts";
import { runImport, type ImportSource } from "./Jobs/Import.ts";
import { runPurgeJob } from "./Jobs/Purge.ts";
import { applyDelta } from "./Protocol/Delta.ts";
import {
bytesToHex,
concatBytes,
decodeOfsDeltaOffset,
decodeTypeSize,
encodeCommit,
encodeTypeSize,
hashObject,
isDeltaType,
isOid,
makeSha1,
ObjectType,
objectTypeName,
parseCommit,
parseTag,
parseTree,
treeEntryKind,
utf8Encode,
ZERO_OID,
type ObjectTypeName,
type Oid,
} from "./Protocol/ObjectCodec.ts";
import * as PackParser from "./Protocol/PackParser.ts";
import type { RandomAccess } from "./Protocol/PackParser.ts";
import * as PartialScan from "./Protocol/PartialScan.ts";
import type { ScanResult } from "./Protocol/PartialScan.ts";
import type { DeltaBase, DeltaJob, UnresolvedDelta } from "./Protocol/PartialScan.ts";
import {
decodePktLines,
errPkt,
flushPkt,
pktPayloadText,
pktText,
readPktLineAt,
type PktLine,
} from "./Protocol/Pkt.ts";
import { progressMessage, pumpPackBody } from "./Protocol/Sideband.ts";
import { StoreError, type ManifestEntry, type ObjectSource } from "./Protocol/Store.ts";
import {
applyTreeChanges,
conflictingPaths,
diffTrees,
type DiffEntryData,
} from "./Protocol/TreeDiff.ts";
import * as Zlib from "./Protocol/Zlib.ts";
import { decodeStagedBatch } from "./PushWire.ts";
import { RegistryStore, ulid } from "./RegistryObject.ts";
import { computeClosure } from "./Store/Closure.ts";
import { encodeHeadSnapshot, type HeadSnapshot } from "./Store/HeadSnapshot.ts";
import { SPILL_PART_BYTES } from "./Store/IncomingBody.ts";
import { headKey, packKeyOf } from "./Store/Keys.ts";
import {
makeObjectStore,
MAX_OBJECT_SIZE,
type ObjectStore,
type StagedObject,
LIVE_OBJECTS,
} from "./Store/ObjectStore.ts";
import { blobRandomAccess } from "./Store/PackSource.ts";
import {
initRepoSchema,
makeSqlClient,
type JobRow,
type PullRow,
type RefRow,
} from "./Store/Sql.ts";
import type { StreamingFeeder } from "./Store/StreamingSource.ts";
import { BACKPRESSURE_BYTES, RETAIN_BYTES } from "./Store/StreamingSource.ts";
// ─────────────────────────────────────────────────────────────────────────────
// Constants
// ─────────────────────────────────────────────────────────────────────────────
/**
* Pushes up to this size are ingested straight from memory; larger ones are
* parked in R2 and parsed from there (DESIGN.md §3.6). A **buffering
* threshold, not a limit** — a push larger than this is no longer rejected.
*
* The spill threshold of the streaming receive (`receiveWireBody`): bodies
* up to this size are buffered and parsed from memory (the fast path, at
* 50 MiB the bound v1 validated for a 128 MB isolate); larger bodies are
* never materialized — they stream into an R2 multipart upload as they
* arrive and are parsed back through bounded windows.
*
* Measured on the 38 MiB alchemy-repo push: ~20 s in memory, ~37 s through
* R2. The R2 path costs about 2x, and is unbounded — the full 67 MiB
* alchemy history (44k objects) pushed in 92 s.
*/
export const MAX_PACK_BYTES = 4 * 1024 * 1024;
// 4 MiB (was 50, then 24): with promoted wire packs (DESIGN §22.5) the
// spilled path is the FAST one — it never writes blob bytes to SQLite, so
// staging and the staged→live flip cost milliseconds instead of seconds
// (a 5.7 MiB in-memory push spent 6 s in finalize on production; a 40 MiB
// spilled one 0.7 s). Only genuinely small pushes stay in memory.
// 24 MiB, down from the 50 MiB v1 validated with ONE active repo: instances
// of this class share an isolate, so the in-memory pack, the ingest's
// resolved-content LRU (20 MiB) and staging batch (8 MiB) sit beside every
// other active repo's working set. DESIGN §22.4 records the OOM that
// motivated it; the R2 path costs ~2x on the bytes above the threshold.
/** Objects staged per SQL transaction during ingest (DESIGN.md §16.6). */
/** Bytes per hasher part — the spill part size, so a part is one R2 part (DESIGN §22.7). */
export const HASH_PART_BYTES = 4 * 1024 * 1024;
/** Hasher calls in flight per push (DESIGN §22.8). */
export const HASH_CONCURRENCY = 8;
// 4 MiB (was 8): the hasher chain is sequential (each part needs the
// previous part's boundary), so a smaller part shortens both the tail after
// the upload ends and the granularity at which the upload can run ahead.
export const STAGE_BATCH_OBJECTS = 2048;
// 2048 (was 256): measured on production, staging cost tracked the number
// of transaction COMMITs (~100 ms each), not rows — 61 commits for a
// 15.6k-object push. Objects average a few KiB, so 2048 stays well under
// STAGE_BATCH_BYTES; the byte cap still bounds memory for large objects.
/** Byte budget per staging batch, so a batch of large blobs stays bounded. */
export const STAGE_BATCH_BYTES = 8 * 1024 * 1024;
// The tree-diff caps (`MAX_DIFF_FILES`, `MAX_DIFF_TREE_READS`) live with
// the pure walk in `Protocol/TreeDiff.ts`; re-exported here alongside the other
// serving-plane bounds.
export { MAX_DIFF_FILES, MAX_DIFF_TREE_READS } from "./Protocol/TreeDiff.ts";
/** Max commits popped by the merge-base paint walk. */
export const MAX_COMPARE_WALK = 10_000;
/** Max commits returned in a Comparison's `commits` array. */
export const MAX_COMPARE_COMMITS = 250;
/**
* Max commits popped by the PR live-field walk (`getPull`). Saturation
* makes the numeric detail fields `null` (reason `"unknown"`); merge still
* works because {@link MAX_PULL_MERGE_WALK} gives the merge path a deeper
* budget.
*/
export const MAX_PULL_WALK = 5_000;
/** Deeper walk budget for `mergePull` (needs only reachability + one base). */
export const MAX_PULL_MERGE_WALK = 50_000;
/** Conflicting paths reported on a `MergeConflict` (the set may be larger). */
export const MAX_CONFLICT_PATHS = 20;
/** Agent string advertised on the wire. */
export const GIT_AGENT = "git-service/1";
/** How long a push waits for ingest memory before answering 503. */
export const PUSH_WAIT_TIMEOUT = "30 seconds";
/**
* Memory the Repo DO lends to concurrent push ingests, in MiB — one
* permit per MiB (DESIGN.md §2.1).
*
* The semaphore exists for MEMORY, not correctness: a DO is
* single-threaded, but its input gate is only closed across *storage*
* awaits, and receive-pack awaits the network and blob storage — so two
* pushes genuinely interleave. Correctness across that interleave is
* already handled by data scoping (`staged_push = pushId`, and every
* read filtering `staged_push IS NULL`) plus a synchronous
* `transactionSync` finalize. What is left is that two concurrent
* 50 MiB buffers would exhaust the 128 MB isolate.
*
* Bounding the resource rather than the count is what lets pushes to
* different branches proceed together: a normal push is kilobytes and
* takes one permit, so it no longer queues behind a large one.
*/
export const PUSH_MEMORY_BUDGET_MB = 64;
let sharedPushGate: Semaphore.Semaphore | undefined;
/**
* The push-admission semaphore, ONE per isolate (DESIGN §21.2, §22.4). DO
* instances of a class share an isolate, so a per-instance gate would let
* N repos each admit a full budget's worth of concurrent pushes.
*/
export const isolatePushGate: Effect.Effect<Semaphore.Semaphore> = Effect.suspend(() =>
sharedPushGate === undefined
? Semaphore.make(PUSH_MEMORY_BUDGET_MB).pipe(
Effect.tap((gate) =>
Effect.sync(() => {
sharedPushGate = gate;
}),
),
)
: Effect.succeed(sharedPushGate),
);
/**
* Permits a push must reserve: its buffered ceiling in MiB. A body
* larger than {@link MAX_PACK_BYTES} spills to blob storage instead of
* growing in memory, so that is the cap; an absent `content-length`
* (chunked upload) is charged the same worst case.
*/
export const pushPermitsFor = (contentLength: number | undefined): number => {
const mib = (bytes: number) => Math.ceil(bytes / (1024 * 1024));
// What a push can hold at once, in MiB — never its body. An unknown
// length is treated as spilled. Memory safety beats concurrency: on a
// 64 MiB budget that admits one large push per isolate alongside small ones.
// Streaming (DESIGN §22.6–7): the feeder retains RETAIN_BYTES behind the
// pump and buffers BACKPRESSURE_BYTES ahead of it, plus one staging batch.
const spilled = mib(RETAIN_BYTES + BACKPRESSURE_BYTES + STAGE_BATCH_BYTES);
// In memory the caches cannot outgrow what the (small) body can inflate
// to, so charge the body twice (compressed + inflated) plus one for the
// batch, rather than the caches' ceilings.
const permits =
contentLength === undefined || !Number.isFinite(contentLength) || contentLength > MAX_PACK_BYTES
? spilled
: mib(Math.max(contentLength, 0)) * 2 + 1;
return Math.max(1, Math.min(PUSH_MEMORY_BUDGET_MB, permits));
};
/** How long crashed push staging survives before the GC alarm reaps it. */
export const STAGING_TTL_MS = 24 * 60 * 60 * 1000;
/** The realm sent on wire 401s. */
export const WWW_AUTHENTICATE = 'Basic realm="git-service"' as const;
/**
* Worker↔DO headers for the clone-bundle splice (DESIGN.md §11): the DO
* answers an eligible clone with these *instead of* the pack, and the
* Worker streams the named R2 object to the client. Only
* {@link BUNDLE_HASH_HEADER} is kept on the client-visible response, as
* observable proof the fast path served the clone.
*/
export const BUNDLE_KEY_HEADER = "x-git-bundle-key" as const;
export const BUNDLE_HASH_HEADER = "x-git-bundle" as const;
export const BUNDLE_COUNT_HEADER = "x-git-bundle-count" as const;
export const BUNDLE_SIDEBAND_HEADER = "x-git-bundle-sideband" as const;
// ─────────────────────────────────────────────────────────────────────────────
// Errors
// ─────────────────────────────────────────────────────────────────────────────
/**
* A push pack failed to ingest (malformed, oversize, checksum mismatch,
* missing thin base, delta error). Reported in-band as `unpack <reason>` +
* per-ref `ng` — never as an HTTP error.
*/
export class PackIngestError extends Schema.TaggedError<PackIngestError>()("PackIngestError", {
reason: Schema.String,
}) {}
/**
* A syntactically valid request that the v0 grammar does not allow
* (missing wants, malformed command lines, unsupported deepen flavors).
* Answered with an `ERR` pkt.
*/
export class WireProtocolError extends Schema.TaggedError<WireProtocolError>()(
"WireProtocolError",
{ reason: Schema.String },
) {}
// ─────────────────────────────────────────────────────────────────────────────
// RPC data shapes (plain serializable data — no Schema classes over RPC)
// ─────────────────────────────────────────────────────────────────────────────
/** Repo metadata as stored in the `config` table. */
export interface RepoMetaData {
readonly repoId: string;
readonly owner: string;
readonly name: string;
readonly defaultBranch: string;
readonly description: string | null;
readonly readOnly: boolean;
/** Anyone can read/clone without a token (GitHub public-repo model). */
readonly public: boolean;
readonly forkOf: string | null;
readonly status: RepoStatus;
readonly createdAt: number;
/** Object storage breakdown (DESIGN.md §12.1). */
readonly objects: ObjectStatsData;
/** Server-side timing of the last push (DESIGN.md §19 phase 0). */
readonly lastPush: PushStatsData | null;
}
/** `beginPush` input: the parsed ref commands (DESIGN §22.10). */
export interface BeginPushInput {
readonly commands: ReadonlyArray<RefCommand>;
}
/** `beginPush` outcome. `denied` is reported per ref (`ng <ref> <reason>`). */
export type BeginPushResult =
| { readonly _tag: "ok"; readonly pushId: string }
| { readonly _tag: "denied"; readonly reason: string };
/** One ref's outcome in a push report. */
export interface PushRefResult {
readonly ref: string;
readonly ok: boolean;
readonly reason?: string | undefined;
}
/** `commitPush` input: what the Worker-side ingest learned (DESIGN §22.10). */
export interface CommitPushInput {
readonly pushId: string;
readonly commands: ReadonlyArray<RefCommand>;
readonly atomic: boolean;
readonly commits: ReadonlyArray<StagedCommitInfo>;
readonly referenced: ReadonlyArray<Oid>;
readonly referencedParents: ReadonlyArray<Oid>;
/** Rows staged as references into the wire pack (it must then outlive the push). */
readonly promoted: number;
readonly stats: {
readonly objects: number;
readonly bytes: number;
readonly ingestMs: number;
readonly stageMs: number;
readonly phases: Record<string, number> | undefined;
};
}
/** `commitPush` outcome: the `unpack` line and per-ref results. */
export interface CommitPushResult {
readonly unpack: string;
readonly results: ReadonlyArray<PushRefResult>;
}
/** Server-side push timing — see the REST `PushStats` schema. */
export interface PushStatsData {
readonly objects: number;
readonly bytes: number;
readonly ingestMs: number;
readonly stageMs: number;
readonly connectivityMs: number;
readonly finalizeMs: number;
readonly totalMs: number;
/** Ingest split by phase (ms) — see the REST `PushStats.phases` caveat. */
readonly phases?: Record<string, number> | undefined;
}
/** Where a repo's objects live — see the REST `ObjectStats` schema. */
export interface ObjectStatsData {
/** Blobs still in SQLite rows, awaiting compaction into an R2 pack. */
readonly loose: number;
/** Commits, trees and tags: always SQLite rows (never packed). */
readonly resident: number;
/** Blobs in R2 packs. */
readonly packed: number;
readonly r2: number;
readonly bytes: number;
}
/** Patch accepted by {@link GitRepoShape.updateRepoMeta}. */
export interface RepoMetaPatch {
readonly description?: string | null | undefined;
/** Must resolve to an existing branch. */
readonly defaultBranch?: string | undefined;
readonly readOnly?: boolean | undefined;
/** Anyone can read/clone without a token when `true`. */
readonly public?: boolean | undefined;
}
/** One ref (peeled target present for annotated tags). */
export interface RefData {
readonly name: string;
readonly oid: string;
readonly peeled?: string | undefined;
}
/** Result of {@link GitRepoShape.listRefs}. */
export interface RefsPage {
/** The default branch's full ref name, `null` on an unborn repo. */
readonly head: string | null;
readonly refs: ReadonlyArray<RefData>;
}
/** Input of {@link GitRepoShape.initRepo} (also fork/import bootstrap). */
export interface InitRepoInput {
readonly repoId: string;
readonly owner: string;
readonly name: string;
readonly defaultBranch: string;
readonly description: string | null;
readonly readOnly: boolean;
/** Anyone can read/clone without a token when `true`. */
readonly public: boolean;
readonly forkOf: string | null;
}
/** Result of init/fork/import bootstrap: the meta + a bootstrap token. */
export interface InitRepoResult {
readonly meta: RepoMetaData;
}
/** Input of {@link GitRepoShape.updateRef}. */
export interface UpdateRefInput {
readonly name: string;
readonly newOid: string;
/**
* CAS token: `string` = must currently equal; `null` = must not exist;
* absent = unconditional.
*/
readonly expectedOid?: string | null | undefined;
}
/** Input of {@link GitRepoShape.removeRef}. */
export interface RemoveRefInput {
readonly name: string;
readonly expectedOid?: string | undefined;
}
/** Input of {@link GitRepoShape.readObject}. */
export interface ReadObjectInput {
readonly oid: string;
/** When set, a different actual type fails with `WrongObjectType`. */
readonly expect?: ObjectTypeName | undefined;
}
/** A fully-read object (content inflated, ≤ 64 MiB by the ingest cap). */
export interface ObjectData {
readonly oid: string;
readonly type: ObjectTypeName;
readonly size: number;
readonly content: Uint8Array;
}
/** One commit for the REST read surface. */
export interface CommitData {
readonly oid: string;
readonly tree: string;
readonly parents: ReadonlyArray<string>;
readonly author: SignatureData;
readonly committer: SignatureData;
readonly message: string;
}
/** An author/committer signature. */
export interface SignatureData {
readonly name: string;
readonly email: string;
readonly date: number;
readonly tz: string;
}
/** Input of {@link GitRepoShape.readCommitLog}. */
export interface CommitLogInput {
/** Refname or 40-hex oid; default = HEAD (the default branch). */
readonly ref?: string | undefined;
readonly cursor?: string | undefined;
readonly limit?: number | undefined;
}
/** One page of {@link GitRepoShape.readCommitLog}. */
export interface CommitLogPage {
readonly items: ReadonlyArray<CommitData>;
readonly nextCursor: string | null;
readonly hasMore: boolean;
}
// `DiffEntryData` (one changed file) is defined next to the pure walk in
// `Protocol/TreeDiff.ts`; re-exported here with the other RPC data shapes.
export type { DiffEntryData } from "./Protocol/TreeDiff.ts";
/** Result of {@link GitRepoShape.readCommitDiff}. */
export interface CommitDiffData {
readonly oid: string;
readonly parent: string | null;
readonly files: ReadonlyArray<DiffEntryData>;
readonly truncated: boolean;
}
/** Input of {@link GitRepoShape.compareCommits}. */
export interface CompareInput {
/** Refname or 40-hex oid. */
readonly base: string;
readonly head: string;
}
/** Result of {@link GitRepoShape.compareCommits}. */
export interface CompareData {
readonly base: string;
readonly head: string;
readonly mergeBase: string;
readonly aheadBy: number;
readonly behindBy: number;
readonly commits: ReadonlyArray<CommitData>;
readonly commitsTruncated: boolean;
readonly files: ReadonlyArray<DiffEntryData>;
readonly filesTruncated: boolean;
}
/** Input of {@link GitRepoShape.readFileAtPath}. */
export interface ReadFileInput {
/** Refname or 40-hex oid; default = HEAD. */
readonly ref?: string | undefined;
/** Slash-separated path inside the tree. */
readonly path: string;
}
/** A file read via tree walk. */
export interface FileData {
readonly oid: string;
/** Octal mode string, e.g. `100644`. */
readonly mode: string;
readonly size: number;
readonly content: Uint8Array;
}
/** One pull-request row (see the REST `Pull` schema). */
export interface PullData {
readonly number: number;
readonly title: string;
readonly body: string | null;
/** Full base ref name, e.g. `refs/heads/main`. */
readonly baseRef: string;
/** Full head ref name, e.g. `refs/heads/feature`. */
readonly headRef: string;
readonly state: "open" | "closed" | "merged";
readonly createdAt: number;
readonly updatedAt: number;
readonly mergedAt: number | null;
/** FF: the head tip; the merge commit otherwise. Set iff state=merged. */
readonly mergeCommit: string | null;
}
/** Detail = row + live compare fields (`null` = uncomputable, see REST docs). */
export interface PullDetailData extends PullData {
readonly baseOid: string | null;
readonly headOid: string | null;
readonly mergeBase: string | null;
readonly aheadBy: number | null;
readonly behindBy: number | null;
readonly mergeable: boolean | null;
readonly mergeableReason: "ff" | "merge-commit" | "conflict" | "up-to-date" | "unknown" | null;
}
/** Input of {@link GitRepoShape.createPull}. */
export interface CreatePullInput {
readonly title: string;
readonly body?: string | undefined;
/** Short or full branch name; normalized to `refs/heads/…` here. */
readonly base: string;
readonly head: string;
}
/** Input of {@link GitRepoShape.listPulls}. */
export interface ListPullsInput {
readonly state?: "open" | "closed" | "merged" | "all" | undefined;
/** Last-seen PR number as a string (keyset pagination, newest first). */
readonly cursor?: string | undefined;
/** Default 20, max 100. */
readonly limit?: number | undefined;
}
/** One page of {@link GitRepoShape.listPulls}. */
export interface PullsPage {
readonly items: ReadonlyArray<PullData>;
readonly nextCursor: string | null;
readonly hasMore: boolean;
}
/** Input of {@link GitRepoShape.updatePull}. */
export interface UpdatePullInput {
readonly number: number;
readonly title?: string | undefined;
/** `null` clears the body. */
readonly body?: string | null | undefined;
readonly state?: "open" | "closed" | undefined;
}
/** Input of {@link GitRepoShape.mergePull}. */
export interface MergePullInput {
readonly number: number;
/** Merge-commit message; ignored on fast-forward. */
readonly message?: string | undefined;
/** Race guard: fail RefConflict when the head tip moved since inspection. */
readonly expectedHeadOid?: string | undefined;
/** Race guard for application inspection of the base branch. */
readonly expectedBaseOid?: string | undefined;
}
/** Result of {@link GitRepoShape.mergePull}. */
export interface MergePullResult {
readonly method: "ff" | "merge-commit";
/** The oid the base ref now points at. */
readonly oid: string;
readonly pull: PullData;
}
/** Input of {@link GitRepoShape.startImport}. */
export interface StartImportInput extends InitRepoInput {
readonly source: ImportSource;
}
/** Input of {@link GitRepoShape.startFork}. */
export interface StartForkInput extends InitRepoInput {
readonly parentRepoId: string;
}
/** The common auth+repo error union of most RPC methods. */
export type RepoAuthError = RepoNotFound | StoreError;
/**
* The Repo DO's typed RPC surface. All methods are `RuntimeContext`-colored
* (callable only through a stub / inside the deployed Worker). Note: no
* method may be named `delete` (Cloudflare stub proxy reserves it).
*/
export interface GitRepoShape {
/** Seeds config (repo create). */
readonly initRepo: (
input: InitRepoInput,
) => Effect.Effect<InitRepoResult, StoreError, RuntimeContext>;
/** Seeds config with `status: importing`, arms the import alarm job. */
readonly startImport: (
input: StartImportInput,
) => Effect.Effect<InitRepoResult, StoreError, RuntimeContext>;
/** Seeds config with `status: forking`, arms the fork alarm job. */
readonly startFork: (
input: StartForkInput,
) => Effect.Effect<InitRepoResult, StoreError, RuntimeContext>;
/** Flips `status: deleting` and arms the purge alarm job. */
/**
* Arms a compaction run now (admin): loose object bytes are moved into an
* immutable R2 pack by the alarm (DESIGN.md §12.1). Normally armed
* automatically by the post-push size check.
*/
readonly startCompact: () => Effect.Effect<void, RepoAuthError, RuntimeContext>;
readonly startPurge: () => Effect.Effect<void, RepoAuthError, RuntimeContext>;
readonly getRepoMeta: () => Effect.Effect<RepoMetaData, RepoAuthError, RuntimeContext>;
/** Repo metadata for the Worker's own bookkeeping. */
readonly readMeta: () => Effect.Effect<RepoMetaData, RepoNotFound | StoreError, RuntimeContext>;
readonly updateRepoMeta: (
patch: RepoMetaPatch,
) => Effect.Effect<RepoMetaData, RepoAuthError | RefNotFound, RuntimeContext>;
readonly listRefs: (prefix?: string) => Effect.Effect<RefsPage, RepoAuthError, RuntimeContext>;
readonly getRef: (
name: string,
) => Effect.Effect<RefData, RepoAuthError | RefNotFound, RuntimeContext>;
readonly updateRef: (
input: UpdateRefInput,
) => Effect.Effect<
RefData,
RepoAuthError | RefConflict | ObjectNotFound | ReadOnlyRepo,
RuntimeContext
>;
readonly removeRef: (
input: RemoveRefInput,
) => Effect.Effect<
void,
RepoAuthError | RefNotFound | RefConflict | ReadOnlyRepo,
RuntimeContext
>;
readonly readObject: (
input: ReadObjectInput,
) => Effect.Effect<ObjectData, RepoAuthError | ObjectNotFound | WrongObjectType, RuntimeContext>;
readonly readCommitLog: (
input: CommitLogInput,
) => Effect.Effect<CommitLogPage, RepoAuthError | RefNotFound, RuntimeContext>;
/**
* The changed-file list of a commit vs its FIRST parent (empty tree for
* a root commit) — see the REST `diff` endpoint.
*/
readonly readCommitDiff: (input: {
readonly oid: string;
}) => Effect.Effect<
CommitDiffData,
RepoAuthError | ObjectNotFound | WrongObjectType,
RuntimeContext
>;
/**
* Three-dot comparison of two revisions: merge base, ahead/behind
* counts, head-side commits, and the mergeBase..head file diff — see
* the REST `compare` endpoint.
*/
readonly compareCommits: (
input: CompareInput,
) => Effect.Effect<
CompareData,
RepoAuthError | RefNotFound | ObjectNotFound | WrongObjectType | NoMergeBase,
RuntimeContext
>;
readonly readFileAtPath: (
input: ReadFileInput,
) => Effect.Effect<FileData, RepoAuthError | RefNotFound | ObjectNotFound, RuntimeContext>;
/** Opens a pull request (same-repo branches, `write` scope). */
readonly createPull: (
input: CreatePullInput,
) => Effect.Effect<
PullData,
RepoAuthError | BranchMissing | PullExists | ValidationError,
RuntimeContext
>;
/** Lists PRs newest-first with keyset pagination (`read` scope). */
readonly listPulls: (
input: ListPullsInput,
) => Effect.Effect<PullsPage, RepoAuthError, RuntimeContext>;
/** Reads one PR with live compare fields recomputed from current tips. */
readonly getPull: (
number: number,
) => Effect.Effect<PullDetailData, RepoAuthError | PullNotFound, RuntimeContext>;
/** Patches title/body, or closes/reopens via `state` (`write` scope). */
readonly updatePull: (
input: UpdatePullInput,
) => Effect.Effect<PullData, RepoAuthError | PullNotFound | PullStateConflict, RuntimeContext>;
/**
* Merges an open PR: fast-forward when possible, else a two-parent merge
* commit iff the three-way tree merge is trivial (no path changed on both
* sides vs the merge base). Conflicts are a typed 409 — the server never
* writes conflict markers.
*/
readonly mergePull: (
input: MergePullInput,
) => Effect.Effect<
MergePullResult,
| RepoAuthError
| PullNotFound
| PullStateConflict
| BranchMissing
| NothingToMerge
| MergeConflict
| RefConflict
| ReadOnlyRepo,
RuntimeContext
>;
/**
* Streams a row snapshot for a fork (DESIGN.md §2.3 Fork). Internal —
* called only by a sibling Repo DO's fork alarm job; never routed from
* the Worker.
*/
readonly snapshotRows: () => Stream.Stream<SnapshotChunk, StoreError>;
/**
* Push, phase 1 (DESIGN §22.10): authorize the parsed commands and open
* a staging push. The Worker then streams the pack through the hasher
* fan-out and the spill, staging rows with `stagePush`.
*/
readonly beginPush: (
input: BeginPushInput,
) => Effect.Effect<BeginPushResult, RepoNotFound | StoreError, RuntimeContext>;
/** Push, phase 2: stage one batch of rows (see `PushWire.ts`). */
readonly stagePush: (
pushId: string,
batch: Uint8Array,
) => Effect.Effect<void, RepoNotFound | StoreError, RuntimeContext>;
/** Abort uncommitted staging; false means committed data must be retained. */
readonly abortPush: (
pushId: string,
packIds?: ReadonlyArray<string>,
) => Effect.Effect<boolean, RepoNotFound | StoreError, RuntimeContext>;
/** Validate connectivity without moving refs. Commit repeats this check. */
readonly validatePush: (
input: CommitPushInput,
) => Effect.Effect<void, RepoNotFound | StoreError, RuntimeContext>;
/** Read from this push's staging area and the live store only. */
readonly readPreparedObject: (
pushId: string,
oid: Oid,
maxBytes: number,
) => Effect.Effect<
{ readonly type: ObjectType; readonly content: Uint8Array } | undefined,
RepoNotFound | StoreError,
RuntimeContext
>;
/** A live object for thin-delta base resolution, or `undefined`. */
readonly readPushBase: (
oid: Oid,
) => Effect.Effect<
{ readonly type: ObjectType; readonly content: Uint8Array } | undefined,
RepoNotFound | StoreError,
RuntimeContext
>;
/**
* Push, phase 3: connectivity over the staged rows, the commit-graph
* rows, the transactional ref CAS, and post-commit bookkeeping.
*/
readonly commitPush: (
input: CommitPushInput,
) => Effect.Effect<CommitPushResult, RepoNotFound | StoreError, RuntimeContext>;
/** The git wire protocol (info/refs, upload-pack). */
readonly fetch: HttpEffect<Cloudflare.DurableObjectState | RuntimeContext>;
/** Alarm job dispatcher (import / fork / purge / gc). */
readonly alarm: (
info?: Cloudflare.AlarmInvocationInfo,
) => Effect.Effect<void, never, RuntimeContext>;
}
// ─────────────────────────────────────────────────────────────────────────────
// Protocol choreography — pure helpers (exported for tests / future split)
// ─────────────────────────────────────────────────────────────────────────────
/** Upload-pack capability list (DESIGN.md §4). */
export const uploadCapabilities = (defaultBranch: string): string =>
`multi_ack_detailed no-done side-band-64k shallow ofs-delta agent=${GIT_AGENT} symref=HEAD:refs/heads/${defaultBranch} object-format=sha1`;
/** Receive-pack capability list (DESIGN.md §4). */
export const receiveCapabilities = (): string =>
`report-status report-status-v2 delete-refs side-band-64k atomic ofs-delta object-format=sha1 agent=${GIT_AGENT}`;
/** The two smart-HTTP services. */
export type GitService = "git-upload-pack" | "git-receive-pack";
/**
* Builds the v0 `info/refs` advertisement (with `# service=` prelude):
* HEAD first with the capability line, refs sorted, annotated tags peeled
* (`^{}` lines), zero-id `capabilities^{}` on an empty repo.
*/
export const buildAdvertisement = (options: {
readonly service: GitService;
readonly defaultBranch: string;
/** Sorted by name; peeled targets included for annotated tags. */
readonly refs: ReadonlyArray<RefData>;
}): Uint8Array => {
const caps =
options.service === "git-upload-pack"
? uploadCapabilities(options.defaultBranch)
: receiveCapabilities();
const parts: Array<Uint8Array> = [pktText(`# service=${options.service}`), flushPkt];
const lines: Array<string> = [];
const headRef = options.refs.find((ref) => ref.name === `refs/heads/${options.defaultBranch}`);
if (options.service === "git-upload-pack" && headRef !== undefined) {
lines.push(`${headRef.oid} HEAD`);
}
for (const ref of options.refs) {
lines.push(`${ref.oid} ${ref.name}`);
if (ref.peeled !== undefined) {
lines.push(`${ref.peeled} ${ref.name}^{}`);
}
}
if (lines.length === 0) {
parts.push(pktText(`${ZERO_OID} capabilities^{}\0${caps}`));
} else {
lines.forEach((line, index) => {
parts.push(index === 0 ? pktText(`${line}\0${caps}`) : pktText(line));
});
}
parts.push(flushPkt);
return concatBytes(parts);
};
/** A parsed v0 upload-pack request. */
export interface UploadPackRequest {
readonly wants: ReadonlyArray<Oid>;
readonly haves: ReadonlyArray<Oid>;
readonly done: boolean;
readonly depth: number | undefined;
readonly clientShallow: ReadonlyArray<Oid>;
readonly capabilities: ReadonlySet<string>;
}
/**
* Parses a v0 upload-pack POST body (already gunzipped) into wants /
* shallow / deepen / haves / done. Rejects `deepen-since` / `deepen-not` /
* `filter` (not advertised) with {@link WireProtocolError}.
*/
export const parseUploadPackRequest = (
pkts: ReadonlyArray<PktLine>,
): Effect.Effect<UploadPackRequest, WireProtocolError> =>
Effect.suspend(() => {
const wants: Array<Oid> = [];
const haves: Array<Oid> = [];
const clientShallow: Array<Oid> = [];
let done = false;
let depth: number | undefined;
let capabilities: ReadonlySet<string> = new Set();
for (const pkt of pkts) {
if (pkt._tag !== "data") continue;
const text = pktPayloadText(pkt.payload);
if (text === "done") {
done = true;
continue;
}
if (text.startsWith("want ")) {
let rest = text.slice(5);
if (wants.length === 0) {
const space = rest.indexOf(" ");
if (space !== -1) {
capabilities = new Set(
rest
.slice(space + 1)
.split(" ")
.filter(Boolean),
);
rest = rest.slice(0, space);
}
}
if (!isOid(rest)) {
return Effect.fail(WireProtocolError.make({ reason: `malformed want line: ${text}` }));
}
wants.push(rest);
continue;
}
if (text.startsWith("have ")) {
const oid = text.slice(5, 45);
if (isOid(oid)) haves.push(oid);
continue;
}
if (text.startsWith("shallow ")) {
const oid = text.slice(8);
if (!isOid(oid)) {
return Effect.fail(
WireProtocolError.make({
reason: `malformed shallow line: ${text}`,
}),
);
}
clientShallow.push(oid);
continue;
}
if (text.startsWith("deepen ")) {
const n = Number.parseInt(text.slice(7), 10);
if (!Number.isInteger(n) || n <= 0) {
return Effect.fail(WireProtocolError.make({ reason: `malformed deepen line: ${text}` }));
}
depth = n;
continue;
}
if (text.startsWith("deepen-since ") || text.startsWith("deepen-not ")) {
return Effect.fail(
WireProtocolError.make({
reason: `${text.split(" ")[0]} is not supported`,
}),
);
}
if (text.startsWith("filter ")) {
return Effect.fail(WireProtocolError.make({ reason: "filter is not supported" }));
}
// ignore unknown lines defensively
}
if (wants.length === 0) {
return Effect.fail(WireProtocolError.make({ reason: "upload-pack request has no wants" }));
}
return Effect.succeed({
wants,
haves,
done,
depth,
clientShallow,
capabilities,
});
});
/** One receive-pack command (`<old> <new> <refname>`). */
export interface RefCommand {
readonly oldOid: Oid;
readonly newOid: Oid;
readonly ref: string;
}
/** A parsed v0 receive-pack request head. */
export interface ReceivePackRequest {
readonly commands: ReadonlyArray<RefCommand>;
readonly capabilities: ReadonlySet<string>;
/** Byte offset of the pack within the body (== length when no pack). */
readonly packStart: number;
/** True for git's bare-flush `http.postBuffer` probe. */
readonly probe: boolean;
}
const REF_NAME_REGEX = /^refs\/[^\s~^:?*[\\]+$/;
/**
* Parses a v0 receive-pack POST body (already gunzipped): command
* pkt-lines (first carries `\0caps`) up to the flush, leaving the raw pack
* bytes at `packStart`. Detects the empty-flush probe (DESIGN.md §2.3
* Push step 2).
*/
export const parseReceivePackRequest = (
body: Uint8Array,
): Effect.Effect<ReceivePackRequest, WireProtocolError> =>
Effect.suspend(() => {
const commands: Array<RefCommand> = [];
let capabilities: ReadonlySet<string> = new Set();
let pos = 0;
let first = true;
for (;;) {
if (pos >= body.length) {
// Body ended without a flush: only legal as the bare probe of an
// entirely empty body.
if (commands.length === 0 && pos === 0) {
return Effect.succeed({
commands: [],
capabilities,
packStart: body.length,
probe: true,
});
}
return Effect.fail(
WireProtocolError.make({
reason: "receive-pack request ended before flush",
}),
);
}
const r = readPktLineAt(body, pos);
if (r._tag === "incomplete" || r._tag === "invalid") {
return Effect.fail(
WireProtocolError.make({
reason: r._tag === "invalid" ? r.reason : "truncated receive-pack request",
}),
);
}
pos = r.next;
if (r.pkt._tag === "flush") {
return Effect.succeed({
commands,
capabilities,
packStart: pos,
probe: commands.length === 0 && pos >= body.length,
});
}
if (r.pkt._tag !== "data") continue;
let text = pktPayloadText(r.pkt.payload);
if (first) {
const nul = text.indexOf("\0");
if (nul !== -1) {
capabilities = new Set(
text
.slice(nul + 1)
.split(" ")
.filter(Boolean),
);
text = text.slice(0, nul);
}
first = false;
}
const oldOid = text.slice(0, 40);
const newOid = text.slice(41, 81);
const ref = text.slice(82);
if (
!isOid(oldOid) ||
!isOid(newOid) ||
text[40] !== " " ||
text[81] !== " " ||
!REF_NAME_REGEX.test(ref)
) {
return Effect.fail(WireProtocolError.make({ reason: `malformed command line: ${text}` }));
}
commands.push({ oldOid, newOid, ref });
}
});
/** The 12-byte pack header: `PACK`, version 2, entry count. */
export const packHeader = (count: number): Uint8Array => {
const out = new Uint8Array(12);
out.set(utf8Encode("PACK"), 0);
const view = new DataView(out.buffer);
view.setUint32(4, 2);
view.setUint32(8, count);
return out;
};
/**
* The §3.5 no-delta pack emitter: header + per-entry
* `varint(type,size) + stored zdata` (verbatim — zero compression CPU) +
* incremental SHA-1 trailer. Constant memory; the manifest is already
* materialized so the count is known upfront.
*/
export const packStream = (
entries: ReadonlyArray<ManifestEntry>,
objects: ObjectSource,
): Stream.Stream<Uint8Array, StoreError> =>
Stream.suspend(() => {
const sha = makeSha1();
const tap = (bytes: Uint8Array): Uint8Array => {
sha.update(bytes);
return bytes;
};
const header = Stream.sync(() => tap(packHeader(entries.length)));
const body = packEntryStream(entries, objects).pipe(Stream.map(tap));
const trailer = Stream.sync(() => sha.digest());
return header.pipe(Stream.concat(body), Stream.concat(trailer));
});
/**
* The entry stream alone — no header, no trailer. Because entries are
* stored non-delta (`varint(type,size) + zdata`), entry streams from
* different sources concatenate into one valid pack under a rewritten
* header and a fresh trailer. That is what lets a clone be served as
* "bundle bytes + a small delta" (DESIGN.md §21.2), and it is the same
* property geometric pack merging relies on.
*/
export const packEntryStream = (
entries: ReadonlyArray<ManifestEntry>,
objects: ObjectSource,
): Stream.Stream<Uint8Array, StoreError> =>
objects.packEntries !== undefined
? objects.packEntries(entries)
: Stream.fromIterable(entries).pipe(
Stream.flatMap((entry) =>
Stream.sync(() => encodeTypeSize(entry.type, entry.size)).pipe(
Stream.concat(objects.readZData(entry.oid)),
),
),
);
// ─────────────────────────────────────────────────────────────────────────────
// Pack ingest (DESIGN.md §3.6) — buffered v1, RandomAccess seam is the buffer
// ─────────────────────────────────────────────────────────────────────────────
/** A commit staged by an ingest, with what the commit graph needs. */
export interface StagedCommitInfo {
readonly oid: Oid;
readonly tree: Oid;
readonly parents: ReadonlyArray<Oid>;
readonly commitTime: number;
}
/** The result of a successful pack ingest. */
export interface IngestResult {
/** Objects in the pack (including already-existing re-pushed ones). */
readonly objectCount: number;
/** SQL staging time, when measured by the streaming ingest path. */
readonly stageMs?: number | undefined;
/** Per-phase CPU split of ingest (ms) — see `PackParser` `phases`. */
readonly phases?: Record<string, number> | undefined;
/** Objects staged as references into a promoted wire pack (DESIGN §22.5). */
readonly promoted?: number | undefined;
/** The completed spilled object's key, when the body spilled (DESIGN §22.10). */
readonly parkedKey?: string | undefined;
/** The resolved pack's key — delta-resolved blobs, promoted (DESIGN §22.13). */
readonly resolvedKey?: string | undefined;
/** The pack's trailer checksum (hex). */
readonly packSha: string;
/** Staged commits (commit-graph rows are inserted at finalize). */
readonly commits: ReadonlyArray<StagedCommitInfo>;
/** Oids referenced by staged trees/tags + commit trees (hard edges). */
readonly referenced: ReadonlyArray<Oid>;
/** Oids referenced as commit parents (soft under shallow import). */
readonly referencedParents: ReadonlyArray<Oid>;
}
/** Bounded LRU of inflated contents shared across delta resolution. */
const makeContentCache = (budget: number) => {
const map = new Map<string, Uint8Array>();
let total = 0;
return {
get: (key: string): Uint8Array | undefined => {
const value = map.get(key);
if (value !== undefined) {
map.delete(key);
map.set(key, value);
}
return value;
},
set: (key: string, value: Uint8Array): void => {
if (value.byteLength > budget || map.has(key)) return;
map.set(key, value);
total += value.byteLength;
for (const [k, v] of map) {
if (total <= budget) break;
map.delete(k);
total -= v.byteLength;
}
},
};
};
/** Delta-resolution content cache budget (DESIGN.md §3.6): 20 MiB. */
const CACHE_BUDGET = 20 * 1024 * 1024;
/** Maximum delta-chain depth tolerated (git's own default is 50). */
const MAX_DELTA_DEPTH = 64;
interface RawEntry {
readonly offset: number;
readonly kind: "object" | "ofs" | "ref";
/** Real object type for `kind: "object"`. */
readonly type: ObjectType | undefined;
/** Header size (object size, or delta payload size for deltas). */
readonly declaredSize: number;
/** Where this entry's zlib stream starts. */
readonly zstart: number;
/** Exact compressed span length. */
readonly zlen: number;
readonly baseOffset: number | undefined;
readonly baseOid: Oid | undefined;
/** Filled once resolved. */
oid: Oid | undefined;
resolvedType: ObjectType | undefined;
}
/**
* The store surface ingest needs (DESIGN §22.10): staging rows and thin
* delta bases. The Repo DO satisfies it directly; the Worker satisfies it
* over RPC so the pack never enters the Durable Object.
*/
export interface IngestStore {
readonly insertStagedBatch: ObjectStore["insertStagedBatch"];
/** Awaits staging that `insertStagedBatch` may have left in flight. */
readonly settle?: Effect.Effect<void, StoreError> | undefined;
/** A live base object for a thin delta, or `undefined` when absent. */
readonly readBase: (
oid: Oid,
) => Effect.Effect<
{ readonly type: ObjectType; readonly content: Uint8Array } | undefined,
StoreError
>;
}
/** An {@link IngestStore} over a full {@link ObjectStore}. */
export const ingestStoreOf = (store: ObjectStore): IngestStore => ({
insertStagedBatch: store.insertStagedBatch,
readBase: (oid) =>
store
.getMeta(oid)
.pipe(
Effect.flatMap((meta) =>
meta === undefined
? Effect.undefined
: store.readContent(oid).pipe(Effect.map((content) => ({ type: meta.type, content }))),
),
),
});
/** Options of {@link ingestPack}. */
export interface IngestOptions {
readonly store: IngestStore;
readonly pushId: string;
/**
* When the wire body was spilled to blob storage, stage non-delta blobs
* as references into it (DESIGN §22.5): `packId` is the wire pack's id,
* `base` the pack's byte offset within the spilled object (the request's
* pkt-line commands precede it).
*/
readonly promote?:
| (() => { readonly packId: string; readonly base: number } | undefined)
| undefined;
/**
* When set, entries are inflated and hashed by the hasher service one
* spilled part at a time (DESIGN §22.7) instead of by the in-process
* parser; the DO only pumps bytes and stages rows.
*/
readonly hasher?: HasherShape | undefined;
/** Bytes per hasher part (tests shrink it). @default HASH_PART_BYTES */
readonly partBytes?: number | undefined;
/**
* The streaming spill (DESIGN §22.10): the pump reads the whole request
* BODY in uniform parts, each hasher writes its part to blob storage,
* and the completed object is the promotion target (`packId`) and the
* source's fallback reader. Bodies that end within `threshold` bytes
* stay in memory and nothing is written.
*/
readonly spill?:
| {
readonly body: RandomAccess;
readonly feeder: StreamingFeeder;
readonly packStart: number;
readonly blobs: BlobStoreShape;
readonly key: string;
readonly packId: string;
readonly repoId: string;
readonly threshold: number;
}
| undefined;
/**
* Shallow-import mode: commit parents may be absent (they become walk
* boundaries); trees/blobs/tags must still be fully connected.
*/
readonly allowMissingParents?: boolean | undefined;
}
/**
* Single-pass buffered pack ingest (DESIGN.md §3.6): parse entries,
* store non-delta compressed spans **verbatim**, resolve ofs/ref deltas
* (thin bases from the live store), enforce the 64 MiB object cap, verify
* the trailer SHA-1, and stage everything under `pushId`.
*
* The v1.x streaming upgrade seam: this function's only view of the pack
* is `pack: Uint8Array` random access — swapping in an R2-backed
* `RandomAccess` touches this implementation, not the protocol drivers.
*/
/**
* Ingests a pack from a {@link RandomAccess} source — the path that has **no
* size cap** (DESIGN.md §3.6). Memory is bounded by the source's read
* window and the delta LRU rather than by the pack, so the source can be an
* R2 object of any size (see `Store/PackSource.ts`).
*
* Delegates the wire-format work to the pure `PackParser` (which the pack
* fixtures in `test/pack.test.ts` cover directly) and does the repo-side
* work in its sink: stage the object, and record commit-graph edges.
* Blobs are staged without inflating — only commits, trees and tags need
* their content parsed, and those are small.
*/
export const ingestPackFrom = (
source: RandomAccess,
options: IngestOptions,
): Effect.Effect<IngestResult, PackIngestError | StoreError> =>
Effect.gen(function* () {
const { store, pushId } = options;
const commits: Array<StagedCommitInfo> = [];
const referenced = new Set<Oid>();
const referencedParents = new Set<Oid>();
const record = Effect.fn(function* (oid: Oid, type: ObjectType, content: Uint8Array) {
switch (type) {
case ObjectType.commit: {
const parsed = yield* parseCommit(content).pipe(
Effect.mapError((error) =>
PackIngestError.make({
reason: `bad commit ${oid}: ${error.reason}`,
}),
),
);
commits.push({
oid,
tree: parsed.tree,
parents: parsed.parents,
commitTime: parsed.committer.when,
});
referenced.add(parsed.tree);
for (const parent of parsed.parents) referencedParents.add(parent);
return;
}
case ObjectType.tree: {
const parsed = yield* parseTree(content).pipe(
Effect.mapError((error) =>
PackIngestError.make({
reason: `bad tree ${oid}: ${error.reason}`,
}),
),
);
for (const entry of parsed) {
if (treeEntryKind(entry.mode) !== "commit") {
referenced.add(entry.oid);
}
}
return;
}
case ObjectType.tag: {
const parsed = yield* parseTag(content).pipe(
Effect.mapError((error) =>
PackIngestError.make({
reason: `bad tag ${oid}: ${error.reason}`,
}),
),
);
referenced.add(parsed.object);
return;
}
case ObjectType.blob:
return;
}
});
// Stage in batches: one transaction per batch instead of one (plus an
// existence probe) per object. That per-object cost is what made a
// 13.7k-object push take ~20 s inside a single-threaded DO — the work
// is inherently serial, so the only lever is fewer operations
// (DESIGN.md §16.6).
// Buffered entries keep their pack coordinates so the promote/inline
// decision is made at FLUSH time (DESIGN §22.6): a streaming body only
// becomes a wire pack once it crosses the spill threshold, but the wire
// object holds the body from byte 0, so entries buffered before that
// moment can still be promoted when the flush happens after it.
/** What the parser or the hasher hands over per entry. */
interface Incoming {
readonly oid: Oid;
readonly type: ObjectType;
readonly size: number;
/** Absent for hasher-scanned non-delta entries: read from the source if the row stays inline. */
readonly zdata?: Uint8Array | undefined;
readonly span?: number | undefined;
readonly dataOffset: number;
readonly fromDelta: boolean;
readonly content?: Uint8Array | undefined;
/** Already in a pack in blob storage (the resolved pack, DESIGN §22.13). */
readonly pack?: { readonly packId: string; readonly offset: number } | undefined;
}
let batch: Array<Incoming> = [];
let batchBytes = 0;
let promoted = 0;
/** Time spent in SQL staging; the rest of ingest is CPU. */
let stageMs = 0;
const flush = Effect.fn(function* () {
if (batch.length === 0) return;
const pending = batch;
batch = [];
batchBytes = 0;
const target =
options.promote?.() ??
(spilling && options.spill !== undefined
? { packId: options.spill.packId, base: options.spill.packStart }
: undefined);
const staged: Array<StagedObject> = [];
for (const entry of pending) {
const pack =
entry.pack ??
(target !== undefined &&
!entry.fromDelta &&
entry.type === ObjectType.blob &&
entry.dataOffset >= 0
? { packId: target.packId, offset: target.base + entry.dataOffset }
: undefined);
if (pack !== undefined) promoted += 1;
// A promoted row needs no bytes; an inline row needs its zdata —
// the hasher path did not ship it, so read the span back.
const zdata =
entry.zdata ??
(pack === undefined
? Uint8Array.from(
readSpan?.(entry.dataOffset, entry.span ?? 0) ??
(yield* source.read(entry.dataOffset, entry.span ?? 0)),
)
: new Uint8Array(0));
staged.push({
oid: entry.oid,
type: entry.type,
size: entry.size,
zdata,
zsize: entry.zdata?.byteLength ?? entry.span,
pack,
});
}
const at = yield* Effect.sync(() => performance.now());
yield* store.insertStagedBatch(pushId, staged);
stageMs += yield* Effect.sync(() => performance.now() - at);
});
const phases: Record<string, number> = {};
/** The spill's multipart upload, once the pump decided to spill. */
let upload: BlobMultipart | undefined;
let spilling = false;
/** Set once the spilled object is complete: it is the wire pack. */
let parkedKey: string | undefined;
/** Set once the resolved pack(s) are written (DESIGN §22.13). */
let resolvedKey: string | undefined;
const resolvedKeys: Array<string> = [];
/**
* Delta-resolved blobs held back from staging while the body spills:
* their fresh zlib goes into one resolved pack in blob storage and the
* rows become coordinates, so the DO never writes those bytes.
*/
const resolvedBlobs: Array<Incoming> = [];
/** In-memory span reads over the retained parts (set by the pump). */
let readSpan: ((at: number, length: number) => Uint8Array | undefined) | undefined;
/**
* Stages a run of resolved entries (DESIGN §22.5): the parser's
* synchronous fast path hands over up to SINK_BATCH non-delta entries
* per fiber hop; delta-resolved entries arrive one at a time.
*/
/** Every oid this push stages: connectivity only asks the store about the rest. */
const stagedOids = new Set<Oid>();
const stage = (entries: ReadonlyArray<Incoming>) =>
Effect.gen(function* () {
for (const entry of entries) {
stagedOids.add(entry.oid);
if (
spilling &&
entry.fromDelta &&
entry.type === ObjectType.blob &&
entry.pack === undefined &&
entry.zdata !== undefined
) {
resolvedBlobs.push(entry);
continue;
}
batch.push(entry);
// Counted as inline until the flush decides; promoted rows carry
// no BLOB, so this over-counts — safe for a memory cap.
batchBytes += entry.zdata?.byteLength ?? entry.span ?? 0;
if (batch.length >= STAGE_BATCH_OBJECTS || batchBytes >= STAGE_BATCH_BYTES) {
yield* flush();
}
if (entry.type !== ObjectType.blob) {
if (entry.content === undefined) {
return yield* PackIngestError.make({
reason: `no content for ${entry.oid} (type ${entry.type})`,
});
}
yield* record(entry.oid, entry.type, entry.content);
}
}
});
/**
* The hasher pipeline (DESIGN §22.7): pump the pack in parts through
* the hasher; every part carries the previous part's incomplete tail.
* The DO's own CPU here is a memcpy, the trailer SHA-1 and staging.
*/
const pumpThroughHasher = (hasher: HasherShape) =>
Effect.gen(function* () {
const spill = options.spill;
// Spill parts (uniform, ≥ 5 MiB for R2) and hash chunks: a hasher
// may want smaller chunks than a part (a Lambda's 6 MB payload);
// chunks then divide parts and the pump uploads the parts itself.
const partBytes =
options.partBytes ?? (spill === undefined ? HASH_PART_BYTES : SPILL_PART_BYTES);
const wanted = hasher.chunkBytes ?? partBytes;
const hashBytes = wanted < partBytes && partBytes % wanted === 0 ? wanted : partBytes;
const asIngest = (error: { readonly _tag: string; readonly reason?: string }) =>
PackIngestError.make({
reason: `${error._tag}${error.reason === undefined ? "" : `: ${error.reason}`}`,
});
const header = yield* source.read(0, 12);
if (
header.length < 12 ||
header[0] !== 0x50 ||
header[1] !== 0x41 ||
header[2] !== 0x43 ||
header[3] !== 0x4b
) {
return yield* PackIngestError.make({ reason: "bad pack magic" });
}
const count = new DataView(header.buffer, header.byteOffset).getUint32(8);
// Trailer SHA-1 over [0, total − 20): hash with a 20-byte lag, so
// the last 20 bytes seen are the trailer itself.
const trailerSha = makeSha1();
let lag = new Uint8Array(0);
const feed = (bytes: Uint8Array) => {
const joined = lag.length === 0 ? bytes : concatBytes([lag, bytes]);
if (joined.length > 20) {
trailerSha.update(joined.subarray(0, joined.length - 20));
lag = Uint8Array.from(joined.subarray(joined.length - 20));
} else {
lag = Uint8Array.from(joined);
}
};
interface Known {
readonly oid: Oid;
readonly type: ObjectType;
readonly dataOffset: number;
readonly span: number;
readonly zdata?: Uint8Array | undefined;
readonly content?: Uint8Array | undefined;
}
const known = new Map<number, Known>(); // by header offset
const knownByOid = new Map<string, Known>();
const unresolved: Array<UnresolvedDelta> = [];
// Raw-chunk dispatch (DESIGN §22.9–10): NO scanning here. Every
// chunk goes to a hasher as it arrives (bounded concurrency), which
// resyncs to the first entry boundary inside the chunk and scans and
// hashes from there — and, when the body spills, writes the chunk
// as its part of the multipart upload. Results are consumed in
// chunk order, concurrently with the receive; the one entry
// straddling each chunk edge is stitched from the neighbouring
// payloads and hashed by a small extra call. A boundary that does
// not line up (a resync false positive) falls back to a sequential
// rescan of that chunk from the known boundary.
const gate = yield* Semaphore.make(hasher.concurrency ?? HASH_CONCURRENCY);
interface Chunk {
readonly index: number;
/** Pack-relative offset of payload[0] (negative for a body-aligned first part). */
readonly base: number;
readonly payload: Uint8Array;
readonly fiber: Fiber.Fiber<HashPartResult, PackIngestError>;
}
const chunks: Array<Chunk> = [];
const bytesFrom = (from: number, to: number): Uint8Array => {
const pieces: Array<Uint8Array> = [];
for (const c of chunks) {
const start = Math.max(from, c.base);
const end = Math.min(to, c.base + c.payload.length);
if (end > start) pieces.push(c.payload.subarray(start - c.base, end - c.base));
}
if (pieces.length === 1) return pieces[0]!;
// Neighbouring chunks are views of one slab: hand out one view.
const first = pieces[0]!;
let contiguous = true;
let end = first.byteOffset + first.byteLength;
for (let i = 1; i < pieces.length; i++) {
const piece = pieces[i]!;
if (piece.buffer !== first.buffer || piece.byteOffset !== end) {
contiguous = false;
break;
}
end += piece.byteLength;
}
return contiguous
? new Uint8Array(first.buffer, first.byteOffset, end - first.byteOffset)
: concatBytes(pieces);
};
// Retained payloads serve every span read (inline rows, delta
// bases and deltas) so nothing goes back to the source or the
// spilled object during ingest.
readSpan = (at, length) => {
const bytes = bytesFrom(at, at + length);
return bytes.length === length ? bytes : undefined;
};
const stageResult = (payload: Uint8Array, base: number, result: ScanResult) =>
Effect.gen(function* () {
const incoming: Array<Incoming> = [];
for (const e of result.entries) {
// A hasher may omit non-blob content to bound its response
// (a Lambda): inflate it here from the fresh zdata or the
// retained span — trees, commits and tags are small.
const content =
e.content ??
(e.type === ObjectType.blob
? undefined
: yield* Zlib.inflate(
e.zdata ??
payload.subarray(e.dataOffset - base, e.dataOffset - base + e.span),
).pipe(Effect.mapError(asIngest)));
const item: Known = {
oid: e.oid,
type: e.type,
dataOffset: e.dataOffset,
span: e.span,
zdata: e.zdata,
content,
};
known.set(e.offset, item);
knownByOid.set(e.oid, item);
incoming.push({
oid: e.oid,
type: e.type,
size: e.size,
zdata:
e.zdata ?? payload.subarray(e.dataOffset - base, e.dataOffset - base + e.span),
span: e.span,
dataOffset: e.dataOffset,
fromDelta: e.zdata !== undefined,
content,
});
}
yield* stage(incoming);
// Batches flush by size (STAGE_BATCH_OBJECTS / _BYTES), not per
// chunk: the retained parts hold every span until ingest ends.
for (const u of result.unresolved) unresolved.push(u);
return result.count;
});
// ── phase 1 (producer): read parts as they arrive, dispatch each ──
// Wall-clock phase split (Date.now advances across awaits; the
// platform's cpuTime is the CPU attribution): `PushStats.phases`.
const pumpStarted = Date.now();
let regionCalls = 0;
// Each part's identity arrives after its upload; joined before
// the multipart upload is completed, never on the scan path.
const parts: Array<Effect.Effect<UploadedPart, PackIngestError>> = [];
let producerDone = false;
const wakers: Array<() => void> = [];
const notifyConsumer = () => {
for (const w of wakers.splice(0)) w();
};
const awaitChunk = (k: number) =>
Effect.suspend(() =>
k < chunks.length || producerDone
? Effect.void
: Effect.callback<void>((resume) => {
wakers.push(() => resume(Effect.void));
}),
);
const availableEnd = () => {
const last = chunks[chunks.length - 1];
return last === undefined ? 12 : last.base + last.payload.length;
};
const dispatch = (
payload: Uint8Array,
opts: {
readonly base: number;
readonly skip: number;
readonly resync: boolean;
readonly partNumber: number;
},
) =>
hasher
.hashPart(payload, {
base: opts.base,
skip: opts.skip,
remaining: count,
maxObjectSize: MAX_OBJECT_SIZE,
resync: opts.resync,
spill:
upload === undefined || spill === undefined || !hasher.writesSpill
? undefined
: {
key: spill.key,
uploadId: upload.uploadId,
partNumber: opts.partNumber,
},
})